def test_plots():
G = ox.graph_from_place('Piedmont, California, USA', network_type='drive', simplify=False)
G2 = ox.simplify_graph(G, strict=False)
# test getting colors
co = ox.get_colors(n=5, return_hex=True)
nc = ox.get_node_colors_by_attr(G2, 'osmid')
ec = ox.get_edge_colors_by_attr(G2, 'length')
# save a plot to disk as png
fig, ax = ox.plot_graph(G, save=True, file_format='png')
# save a plot to disk as svg
G_simplified = ox.simplify_graph(G)
fig, ax = ox.plot_graph(G_simplified, show=False, save=True, close=True, file_format='svg')
G_projected = ox.project_graph(G_simplified)
fig, ax = ox.plot_graph(G_projected)
fig, ax = ox.plot_graph(G_projected, fig_height=5, fig_width=5, margin=0.05, axis_off=False, bgcolor='y',
file_format='png', filename='x', dpi=180, annotate=True, node_color='k', node_size=5,
node_alpha=0.1, node_edgecolor='b', node_zorder=5, edge_color='r', edge_linewidth=2,
edge_alpha=0.1, use_geom=False, show=False, save=True, close=True)
fig, ax = ox.plot_figure_ground(G=G_simplified, file_format='png')
fig, ax = ox.plot_figure_ground(point=(33.694981, -117.841375), file_format='png')
fig, ax = ox.plot_figure_ground(address='Denver, Colorado, USA', file_format='png')
def test_plots():
G = ox.graph_from_place('Piedmont, California, USA', network_type='drive', simplify=False)
G2 = ox.simplify_graph(G, strict=False)
# test getting colors
co = ox.get_colors(n=5, return_hex=True)
nc = ox.get_node_colors_by_attr(G2, 'osmid')
ec = ox.get_edge_colors_by_attr(G2, 'length')
# save a plot to disk as png
fig, ax = ox.plot_graph(G, save=True, file_format='png')
# save a plot to disk as svg
G_simplified = ox.simplify_graph(G)
fig, ax = ox.plot_graph(G_simplified, show=False, save=True, close=True, file_format='svg')
G_projected = ox.project_graph(G_simplified)
fig, ax = ox.plot_graph(G_projected)
fig, ax = ox.plot_graph(G_projected, fig_height=5, fig_width=5, margin=0.05, axis_off=False, bgcolor='y',
file_format='png', filename='x', dpi=180, annotate=True, node_color='k', node_size=5,
node_alpha=0.1, node_edgecolor='b', node_zorder=5, edge_color='r', edge_linewidth=2,
edge_alpha=0.1, use_geom=False, show=False, save=True, close=True)
fig, ax = ox.plot_figure_ground(G=G_simplified, file_format='png')
fig, ax = ox.plot_figure_ground(point=(33.694981, -117.841375), file_format='png')
fig, ax = ox.plot_figure_ground(address='Denver, Colorado, USA', file_format='png')
def graph_from_shapefile(path,
in_crs=None,
custom_standards=None,
name='unnamed',
retain_all=False,
simplify=True):
"""
Create Networkx Graph from the shapeFile provided.
:param path: Path of the shapeFile.
:param in_crs: CRS of the shapeFile. Should be a string in 'EPSG:{number}' format. (Default = 'EPSG:4326')
:param custom_standards: Dict object used to convert non standard properties attributes to standard
example = {
"LINK_ID": "osm_id",
"DIRONSIGN":"oneway"
}
json format Key:Value
where:
Key is the Field Name from the shapeFile.
Value is the standard to convert it to.
See documentation for more.
:param name: Name of the Graph to be created.(default = unnamed)
:param retain_all: if True, return the entire graph even if it is not connected
:param simplify: Simplify the graph. (default = True)
:return: Networkx MultiDiGraph
"""
geojson = shp2geojson(path, custom_standards)
if in_crs is not None:
geojson = coordinate_transform(geojson, in_crs=in_crs)
json_data = geojson2osm_json(geojson)
g = ox.create_graph(json_data, name, retain_all=retain_all)
if simplify is True:
g = ox.simplify_graph(g)
return g
def graph_from_shapefile(path, custom_standards=None, name='unnamed', simplify=True):
"""
Create Networkx Graph from the shapeFile provided.
:param path: Path of the shapeFile.
:param custom_standards: Dict object used to convert non standard properties attributes to standard
example = {
"LINK_ID": "osm_id",
"DIRONSIGN":"oneway"
}
json format Key:Value
where:
Key is the Field Name from the shapeFile.
Value is the standard to convert it to.
See documentation for more.
:param name: Name of the Graph to be created.(default = unnamed)
:param simplify: Simplify the graph. (default = True)
:return: Networkx MultiDiGraph
"""
geojson = shp2geojson(path, custom_standards)
json_data = geojson2osm_json(geojson)
g = ox.create_graph(json_data, name)
if simplify is True:
g = ox.simplify_graph(g)
return g
def composeFiles(fileL):
"""compose a list of graphs from files"""
graphL = []
for f in fileL:
graph1 = ox.load_graphml(filename=f)
composed_G = nx.compose_all(graphL)
simp_G = ox.simplify_graph(composed_G)
return composed_G
def simplify_graph(G):
''' strict mode more consistent with non-important mid nodes.'''
# nc = ['b' if ox.is_endpoint(G, node, strict=True) else 'r' for node in G.nodes()]
# fig, ax = ox.plot_graph(G, node_color=nc, node_zorder=3)
G = ox.simplify_graph(G, strict=True)
if iprint>=2: print("simplify_graph showing simplified graph...")
if Has_Display: fig, ax = ox.plot_graph(G, node_color='b', node_zorder=3)
return G
def load_graph(city_name):
global graph
global city
graph = ox.simplify_graph(ox.graph_from_place(
city_name + " France",
simplify=False,
retain_all=True,
network_type='drive',
))
city = city_name
def bike_network(geometry, name):
G = ox.graph_from_polygon(polygon=geometry,
network_type='bike',
name=name,
retain_all=True,
simplify=False)
#non_cycleways = [(u, v, k) for u, v, k, d in G.edges(keys=True, data=True)
# if not ('cycleway' in d or d['highway'] == 'cycleway')]
#G.remove_edges_from(non_cycleways)
G = ox.simplify_graph(G)
return G
def test_plots():
G = ox.graph_from_place('Piedmont, California, USA',
network_type='drive',
simplify=False)
G2 = ox.simplify_graph(G, strict=False)
ec = ox.get_edge_colors_by_attr(G2, 'length')
fig, ax = ox.plot_graph(G, save=True, file_format='png')
G_simplified = ox.simplify_graph(G)
fig, ax = ox.plot_graph(G_simplified,
show=False,
save=True,
close=True,
file_format='svg')
G_projected = ox.project_graph(G_simplified)
fig, ax = ox.plot_graph(G_projected)
fig, ax = ox.plot_graph(G_projected,
fig_height=5,
fig_width=5,
margin=0.05,
axis_off=False,
bgcolor='y',
file_format='png',
filename='x',
dpi=180,
annotate=True,
node_color='k',
node_size=5,
node_alpha=0.1,
node_edgecolor='b',
node_zorder=5,
edge_color='r',
edge_linewidth=2,
edge_alpha=0.1,
use_geom=False,
show=False,
save=True,
close=True)
def print_vehicles(G, V):
H = ox.simplify_graph(G)
fig, ax = ox.plot_graph(H, show=False, close=False)
for v in V.nodes():
ax.plot(V.nodes[v]['x'], V.nodes[v]['y'], marker='o', color='r')
for s, t in V.edges():
ax.plot([V.nodes[s]['x'], V.nodes[t]['x']],
[V.nodes[s]['y'], V.nodes[t]['y']],
color='r',
linewidth=0.1)
#plt.show()
plt.savefig(f"trento_car_{len(V.nodes())}_{len(V.edges())}.pdf")
def download_cambridge_graph(save_path):
cambridge_ll_bbox = [52.245, 52.150, 0.220, 0.025]
raw_graph = ox.graph_from_bbox(*cambridge_ll_bbox,
truncate_by_edge=True,
simplify=False,
network_type='drive')
projected_graph = ox.project_graph(raw_graph)
simplified_graph = ox.simplify_graph(projected_graph)
write_gpickle(simplified_graph, save_path)
def get_graph(filename):
grabber = OSMloader()
p = OSMParser(concurrency=1,
coords_callback=grabber.nodes,
ways_callback=grabber.ways)
p.parse(filename)
grabber.spinner.finish()
print ''
print 'Simplifying graph... '
graph = max(nx.strongly_connected_component_subgraphs(grabber.graph,
copy=True),
key=len)
return nx.DiGraph(simplify_graph(graph, strict=False))
def getBike(self, place):
useful_tags = ox.settings.useful_tags_way + ['cycleway'] + ['bicycle'] + ['oneway'] + ['bicycle:lanes'] + ['cycleway:right'] + ['cycleway:left'] + ['bicycle_road']
ox.utils.config(use_cache=True, log_console=True, useful_tags_way=useful_tags)
G = ox.graph_from_place(query=place, network_type='bike', simplify=False, retain_all=True)
attributes = []
non_cyc = []
for u,v,k,d in G.edges(keys=True, data=True):
for a in d:
name=str(a)+": "+str(d[a])
excluded = ['osmid', 'ref', 'length', 'width', 'name']
if a in excluded:
pass
elif name not in attributes:
attributes.append(name)
bi = False
if "bicycle" in d:
if d['bicycle']=='designated' or d['bicycle']=='official' or d['bicycle']=='track' or d['bicycle']=='use_sidepath':
bi = True
if "bicycle:lanes" in d:
if "designated" in d['bicycle:lanes']:
bi = True
if d['highway']=='cycleway':
pass
elif 'cycleway' in d and d['cycleway'] != "no" and d['cycleway'] != "opposite":
pass
elif 'cycleway:right' in d and d['cycleway:right'] != "no":
pass
elif 'cycleway:left' in d and d['cycleway:left'] != "no":
pass
elif 'bicycle_road' in d and d['bicycle_road'] == "yes":
pass
elif bi:
pass
else:
non_cyc.append((u,v,k))
G.remove_edges_from(non_cyc)
G = ox.utils_graph.remove_isolated_nodes(G)
try:
G = ox.simplify_graph(G)
except:
pass
attributes.sort()
#print(attributes)
return G
def bike_network(city):
try:
G = ox.graph_from_place(city,
network_type='bike',
simplify=False,
which_result=1)
except:
G = ox.graph_from_place(city,
network_type='bike',
simplify=False,
which_result=2)
#non_cycleways = [(u, v, k) for u, v, k, d in G.edges(keys=True, data=True)
# if not ('cycleway' in d or d['highway'] == 'cycleway')]
#G.remove_edges_from(non_cycleways)
#G = ox.remove_isolated_nodes(G)
G = ox.simplify_graph(G)
return ox.project_graph(G)
def return_links_of_interest(graph):
'''
Due to the graph network coming straight from osm having nodes at every bend
in the road, there are far too many roads. Moreover, many roads are quite short
and so will almost always take less than 15 seconds to traverse. Considering that
our measurements are spaced 15 seconds apart, we only want to learn the pdfs
for roads that take around or more than 15 seconds to traverse as we can't exatly
be sure how long it took to traverse shorter roads. Thus, we only consider modelling
a subset of the links of the network known as 'links of interest'.
Parameters:
----------
Graph (networkx multidigraph): Graph that you want to get the links_of_interest from
Returns:
--------
links_of_interest (dictionary): mapping from these 'links of interest' to their corresponding
indices (just an integer from 0-len(links_of_interest)).
'''
# speed limit on inner city roads is 50 mph in Porto.
# We want the time taken on links of interest to be around 15 seconds
# to reduce our measurement error.
# Therefore the minimum length of road we are interested in, assuming
# a min cutoff time of 10 seconds and an average speed of
# 30 mph is (30*1.6)/3600)*10*1000 ~= 130m
# use 100m just to be safe
# This get's rid of roughly half the edges
#we want to use the simplified graph as regular graph has too many edges
simple_graph = ox.simplify_graph(graph, strict=True)
cutoff = 100
edges = list(simple_graph.edges(data='length'))
edges_of_interest = {}
i = 0
for edge_data in edges:
edge_name = get_edge_name(edge_data[0], edge_data[1])
edge_length = edge_data[2]
if edge_length > cutoff:
#edge[0] and edge[1] correspond to the nodes
if edge_name not in edges_of_interest:
# edges are repeated in edges for some reason
edges_of_interest[edge_name] = i
i += 1
return edges_of_interest
def simple_get_roads(city):
"""
Use osmnx to get a simplified version of open street maps for the city
Writes osm_nodes and osm_ways shapefiles to MAP_FP
Args:
city
Returns:
None, but creates the following shape files:
osm_ways.shp - the simplified road network
osm_nodes.shp - the intersections and dead ends
And creates the following directory:
all_nodes - containing edges and nodes directories
for the unsimplified road network
"""
G1 = ox.graph_from_place(city, network_type='drive', simplify=False)
G = ox.simplify_graph(G1)
# Label endpoints
streets_per_node = ox.count_streets_per_node(G)
for node, count in list(streets_per_node.items()):
if count <= 1:
G.nodes()[node]['dead_end'] = True
# osmnx creates a directory for the nodes and edges
# Store all nodes, since they can be other features
ox.save_graph_shapefile(G1, filename='all_nodes', folder=MAP_FP)
# Store simplified network
ox.save_graph_shapefile(G, filename='temp', folder=MAP_FP)
# Copy and remove temp directory
tempdir = os.path.join(MAP_FP, 'temp')
for filename in os.listdir(os.path.join(tempdir, 'edges')):
name, extension = filename.split('.')
shutil.move(os.path.join(tempdir, 'edges', filename),
os.path.join(MAP_FP, 'osm_ways.' + extension))
for filename in os.listdir(os.path.join(tempdir, 'nodes')):
name, extension = filename.split('.')
shutil.move(os.path.join(tempdir, 'nodes', filename),
os.path.join(MAP_FP, 'osm_nodes.' + extension))
shutil.rmtree(tempdir)
def simplifyGraph(G, connection="strong"):
"""simplify a graph"""
# G = ox.load_graphml(filename=fName)
print("G nodes: " + str(len(G.nodes())))
print("G edges: " + str(len(G.edges())))
G_simp = ox.simplify_graph(G)
print('simp_G:')
print("G simp nodes: " + str(len(G_simp.nodes())))
print("G simp edges: " + str(len(G_simp.edges())))
if connection == "strong":
G_connect = max(nx.weakly_connected_component_subgraphs(G_simp), key=len)
elif connection == "weak":
G_connect = max(nx.strongly_connected_component_subgraphs(G_simp), key=len)
elif connection == "undirect":
G_connect = max(nx.connected_component_subgraphs(G_simp.to_undirected()), key=len)
else:
G_connect = G_simp
print('connected_G:')
print("G connected nodes: " + str(len(G_connect.nodes())))
print("G connected edges: " + str(len(G_connect.edges())))
G_proj = ox.project_graph(G_connect, to_crs={'init': 'epsg:4326'})
print('proj done')
return G_proj
def wkt_to_G(wkt_list, im_file=None, min_subgraph_length_pix=30,
min_spur_length_m=5, simplify_graph=True, verbose=False):
'''Execute all functions'''
t0 = time.time()
print ("Running wkt_list_to_nodes_edges()...")
node_loc_dic, edge_dic = wkt_list_to_nodes_edges(wkt_list)
t1 = time.time()
print ("Time to run wkt_list_to_nodes_egdes():", t1 - t0, "seconds")
#print ("node_loc_dic:", node_loc_dic)
#print ("edge_dic:", edge_dic)
print ("Creating G...")
G0 = nodes_edges_to_G(node_loc_dic, edge_dic)
print (" len(G.nodes():", len(G0.nodes()))
print (" len(G.edges():", len(G0.edges()))
#for edge_tmp in G0.edges():
# print ("\n 0 wtk_to_G():", edge_tmp, G0.edge[edge_tmp[0]][edge_tmp[1]])
t2 = time.time()
print ("Time to run nodes_edges_to_G():", t2-t1, "seconds")
print ("Clean out short subgraphs")
G0 = clean_sub_graphs(G0, min_length=min_subgraph_length_pix,
max_nodes_to_skip=30,
weight='length_pix', verbose=True,
super_verbose=False)
t3 = time.time()
print ("Time to run clean_sub_graphs():", t3-t2, "seconds")
if len(G0) == 0:
return G0
# print ("Simplifying graph")
# G0 = ox.simplify_graph(G0.to_directed())
# G0 = G0.to_undirected()
# #G0 = ox.project_graph(G0)
# #G_p_init = create_edge_linestrings(G_p_init, remove_redundant=True, verbose=False)
# t3 = time.time()
# print (" len(G.nodes():", len(G0.nodes()))
# print (" len(G.edges():", len(G0.edges()))
# print ("Time to run simplify graph:", t30 - t3, "seconds")
#for edge_tmp in G0.edges():
# print ("\n 1 wtk_to_G():", edge_tmp, G0.edge[edge_tmp[0]][edge_tmp[1]])
#edge_tmp = G0.edges()[5]
#print (edge_tmp, "G0.edge props:", G0.edge[edge_tmp[0]][edge_tmp[1]])
# geo coords
if im_file:
print ("Running get_node_geo_coords()...")
G1 = get_node_geo_coords(G0, im_file, verbose=verbose)
t4= time.time()
print ("Time to run get_node_geo_coords():", t4-t3, "seconds")
print ("Running get_edge_geo_coords()...")
G1 = get_edge_geo_coords(G1, im_file, verbose=verbose)
t5 = time.time()
print ("Time to run get_edge_geo_coords():", t5-t4, "seconds")
print("pre projection...")
node = list(G1.nodes())[-1]
print(node, "random node props:", G1.nodes[node])
# print an edge
edge_tmp = list(G1.edges())[-1]
print(edge_tmp, "random edge props:", G1.get_edge_data(edge_tmp[0], edge_tmp[1]))
print ("projecting graph...")
G_projected = ox.project_graph(G1)
print("post projection...")
node = list(G_projected.nodes())[-1]
print(node, "random node props:", G_projected.nodes[node])
# print an edge
edge_tmp = list(G_projected.edges())[-1]
print(edge_tmp, "random edge props:", G_projected.get_edge_data(edge_tmp[0], edge_tmp[1]))
t6 = time.time()
print ("Time to project graph:", t6-t5, "seconds")
# simplify
#G_simp = ox.simplify_graph(G_projected.to_directed())
#ox.plot_graph(G_projected)
#G1.edge[19][22]
Gout = G_projected #G_simp
else:
Gout = G0
###########################################################################
# remove short edges
t31 = time.time()
Gout = remove_short_edges(Gout, min_spur_length_m=min_spur_length_m)
t32 = time.time()
print("Time to remove_short_edges():", t32 - t31, "seconds")
###########################################################################
if simplify_graph:
print ("Simplifying graph")
t7 = time.time()
G0 = ox.simplify_graph(Gout.to_directed())
Gout = G0.to_undirected()
#Gout = ox.project_graph(G0)
t8 = time.time()
print ("Time to run simplify graph:", t8-t7, "seconds")
# When the simplify funciton combines edges, it concats multiple
# edge properties into a list. This means that 'geometry_pix' is now
# a list of geoms. Convert this to a linestring with
# shaply.ops.linemergeconcats
print ("Merge 'geometry' linestrings...")
keys_tmp = ['geometry_pix', 'geometry_latlon_wkt', 'geometry_utm_wkt']
for key_tmp in keys_tmp:
print ("Merge", key_tmp, "...")
for i,(u,v,attr_dict) in enumerate(Gout.edges(data=True)):
if (i % 10000) == 0:
print (i, u , v)
geom = attr_dict[key_tmp]
#print (i, u, v, "geom:", geom)
#print (" type(geom):", type(geom))
if type(geom) == list:
# check if the list items are wkt strings, if so, create
# linestrigs
if (type(geom[0]) == str):# or (type(geom_pix[0]) == unicode):
geom = [shapely.wkt.loads(ztmp) for ztmp in geom]
# merge geoms
#geom = shapely.ops.linemerge(geom)
#attr_dict[key_tmp] = geom
attr_dict[key_tmp] = shapely.ops.linemerge(geom)
elif type(geom) == str:
attr_dict[key_tmp] = shapely.wkt.loads(geom)
else:
pass
# assign 'geometry' tag to geometry_utm_wkt
for i,(u,v,attr_dict) in enumerate(Gout.edges(data=True)):
if verbose:
print ("Create 'geometry' field in edges...")
#geom_pix = attr_dict[key_tmp]
line = attr_dict['geometry_utm_wkt']
if type(line) == str:# or type(line) == unicode:
attr_dict['geometry'] = shapely.wkt.loads(line)
else:
attr_dict['geometry'] = attr_dict['geometry_utm_wkt']
# update wkt_pix?
#print ("attr_dict['geometry_pix':", attr_dict['geometry_pix'])
attr_dict['wkt_pix'] = attr_dict['geometry_pix'].wkt
# update 'length_pix'
attr_dict['length_pix'] = np.sum([attr_dict['length_pix']])
# Gout = ox.project_graph(Gout)
# get a few stats (and set to graph properties)
logger1.info("Number of nodes: {}".format(len(Gout.nodes())))
logger1.info("Number of edges: {}".format(len(Gout.edges())))
#print ("Number of nodes:", len(Gout.nodes()))
#print ("Number of edges:", len(Gout.edges()))
Gout.graph['N_nodes'] = len(Gout.nodes())
Gout.graph['N_edges'] = len(Gout.edges())
# get total length of edges
tot_meters = 0
for i,(u,v,attr_dict) in enumerate(Gout.edges(data=True)):
tot_meters += attr_dict['length']
print ("Length of edges (km):", tot_meters/1000)
Gout.graph['Tot_edge_km'] = tot_meters/1000
print ("G.graph:", Gout.graph)
t7 = time.time()
print ("Total time to run wkt_to_G():", t7-t0, "seconds")
#for edge_tmp in Gout.edges():
# print ("\n 2 wtk_to_G():", edge_tmp, Gout.edge[edge_tmp[0]][edge_tmp[1]])
return Gout
def simple_get_roads(config, mapfp):
"""
Use osmnx to get a simplified version of open street maps for the city
Writes osm_nodes and osm_ways shapefiles to mapfp
Args:
config object
Returns:
None
This function creates the following files
features.geojson - traffic signals, crosswalks and intersections
osm_ways.shp - the simplified road network
osm_nodes.shp - the intersections and dead ends
"""
ox.settings.useful_tags_path.append('cycleway')
G1 = get_graph(config)
G = ox.simplify_graph(G1)
# Label endpoints
streets_per_node = ox.count_streets_per_node(G)
for node, count in list(streets_per_node.items()):
if count <= 1:
G.nodes()[node]['dead_end'] = True
G1.nodes()[node]['dead_end'] = True
# osmnx creates a directory for the nodes and edges
# Store all nodes, since they can be other features
# Get relevant node features out of the unsimplified graph
nodes, data = zip(*G1.nodes(data=True))
gdf_nodes = geopandas.GeoDataFrame(list(data), index=nodes)
node_feats = gdf_nodes[gdf_nodes['highway'].isin(
['crossing', 'traffic_signals'])]
intersections = gdf_nodes[gdf_nodes['dead_end'] == True]
names = {'traffic_signals': 'signal', 'crossing': 'crosswalk'}
features = []
for _, row in node_feats.iterrows():
features.append(
geojson.Feature(
geometry=geojson.Point((row['x'], row['y'])),
id=row['osmid'],
properties={'feature': names[row['highway']]},
))
for _, row in intersections.iterrows():
features.append(
geojson.Feature(
geometry=geojson.Point((row['x'], row['y'])),
id=row['osmid'],
properties={'feature': 'intersection'},
))
features = geojson.FeatureCollection(features)
with open(os.path.join(mapfp, 'features.geojson'), "w") as f:
json.dump(features, f)
# Store simplified network
ox.save_graph_shapefile(G, filename='temp', folder=mapfp)
# Copy and remove temp directory
tempdir = os.path.join(mapfp, 'temp')
for filename in os.listdir(os.path.join(tempdir, 'edges')):
_, extension = filename.split('.')
shutil.move(os.path.join(tempdir, 'edges', filename),
os.path.join(mapfp, 'osm_ways.' + extension))
for filename in os.listdir(os.path.join(tempdir, 'nodes')):
_, extension = filename.split('.')
shutil.move(os.path.join(tempdir, 'nodes', filename),
os.path.join(mapfp, 'osm_nodes.' + extension))
shutil.rmtree(tempdir)
def graph_from_jsons(response_jsons,
network_type='all_private',
simplify=True,
retain_all=False,
truncate_by_edge=False,
name='unnamed',
timeout=180,
memory=None,
max_query_area_size=50 * 1000 * 50 * 1000,
clean_periphery=True,
infrastructure='way["highway"]'):
"""
Create a networkx graph from OSM data within the spatial boundaries of the passed-in shapely polygon.
This is a modified routine from osmnx
Parameters
----------
response_jsons : list of responses from osmnx
the shape to get network data within. coordinates should be in units of
latitude-longitude degrees.
network_type : string
what type of street network to get
simplify : bool
if true, simplify the graph topology
retain_all : bool
if True, return the entire graph even if it is not connected
truncate_by_edge : bool
if True retain node if it's outside bbox but at least one of node's
neighbors are within bbox
name : string
the name of the graph
timeout : int
the timeout interval for requests and to pass to API
memory : int
server memory allocation size for the query, in bytes. If none, server
will use its default allocation size
max_query_area_size : float
max size for any part of the geometry, in square degrees: any polygon
bigger will get divided up for multiple queries to API
clean_periphery : bool
if True (and simplify=True), buffer 0.5km to get a graph larger than
requested, then simplify, then truncate it to requested spatial extent
infrastructure : string
download infrastructure of given type (default is streets (ie, 'way["highway"]') but other
infrastructures may be selected like power grids (ie, 'way["power"~"line"]'))
Returns
-------
networkx multidigraph
"""
if clean_periphery and simplify:
g_buffered = ox.create_graph(response_jsons,
name=name,
retain_all=True,
network_type=network_type)
# simplify the graph topology
g = ox.simplify_graph(g_buffered)
# count how many street segments in buffered graph emanate from each
# intersection in un-buffered graph, to retain true counts for each
# intersection, even if some of its neighbors are outside the polygon
g.graph['streets_per_node'] = ox.count_streets_per_node(
g, nodes=g.nodes())
else:
# create the graph from the downloaded data
g = ox.create_graph(response_jsons,
name=name,
retain_all=True,
network_type=network_type)
# simplify the graph topology as the last step. don't truncate after
# simplifying or you may have simplified out to an endpoint beyond the
# truncation distance, in which case you will then strip out your entire
# edge
if simplify:
g = ox.simplify_graph(g)
return g
edge_width=1,
x_lim=(min_x, max_x),
y_lim=(min_y, max_y),
dpi=200,
path=_path)
# Let's use OSMnx to fetch an OSM graph
# We'll use the same raw network for both workflows (hence simplify=False)
multi_di_graph_raw = ox.graph_from_point((lat, lng), dist=1250, simplify=False)
# Workflow 1: Using OSMnx for simplification
# ==========================================
# explicit simplification via OSMnx
multi_di_graph_utm = ox.project_graph(multi_di_graph_raw)
multi_di_graph_simpl = ox.simplify_graph(multi_di_graph_utm)
multi_di_graph_cons = ox.consolidate_intersections(multi_di_graph_simpl,
tolerance=10,
dead_ends=True)
# let's use the same plotting function for both scenarios to aid visual comparisons
multi_graph_cons = graphs.nX_from_OSMnx(multi_di_graph_cons, tolerance=50)
simple_plot(multi_graph_cons, 'images/osmnx_simplification.png')
# WORKFLOW 2: Using cityseer for simplification
# =============================================
# let's convert the OSMnx graph to cityseer compatible `multiGraph`
G_raw = graphs.nX_from_OSMnx(multi_di_graph_raw)
# convert to UTM
G = graphs.nX_wgs_to_utm(G_raw)
# infer geoms
G = graphs.nX_simple_geoms(G)
def wkt_to_G(params):
'''Convert wkt to G with geospatial info.'''
wkt_list, im_file, min_subgraph_length_pix, \
node_iter, edge_iter, \
simplify_graph, \
rdp_epsilon,\
manually_reproject_nodes, \
out_file, pickle_protocol, \
n_threads, verbose \
= params
node_loc_dic, edge_dic = wkt_list_to_nodes_edges(wkt_list,
node_iter=node_iter,
edge_iter=edge_iter)
G0 = nodes_edges_to_G(node_loc_dic, edge_dic)
# This graph will have a unique edge for each line segment, meaning that
# many nodes will have degree 2 and be in the middle of a long edge.
# run clean_sub_graph() in 04_skeletonize.py? - Nope, do it here
# so that adding small terminals works better...
G1 = clean_sub_graphs(G0,
min_length=min_subgraph_length_pix,
weight='length_pix',
verbose=verbose,
super_verbose=False)
if len(G1) == 0:
return G1
# geo coords
if im_file:
G1 = get_node_geo_coords(G1, im_file, verbose=verbose)
G1 = get_edge_geo_coords(G1, im_file, verbose=verbose)
node = list(G1.nodes())[-1]
if verbose:
print(node, "random node props:", G1.nodes[node])
# print an edge
edge_tmp = list(G1.edges())[-1]
print(edge_tmp, "random edge props:",
G1.get_edge_data(edge_tmp[0], edge_tmp[1]))
G_projected = ox.project_graph(G1)
# get geom wkt (for printing/viewing purposes)
for i, (u, v, attr_dict) in enumerate(G_projected.edges(data=True)):
# attr_dict['geometry_wkt'] = attr_dict['geometry'].wkt # broken
attr_dict['geometry_wkt'] = attr_dict['geometry_utm_wkt']
if verbose:
node = list(G_projected.nodes())[-1]
print(node, "random node props:", G_projected.nodes[node])
# print an edge
edge_tmp = list(G_projected.edges())[-1]
print(edge_tmp, "random edge props:",
G_projected.get_edge_data(edge_tmp[0], edge_tmp[1]))
Gout = G_projected
else:
Gout = G0
if simplify_graph:
# 'geometry' tag breaks simplify, so make it a wkt
for i, (u, v, attr_dict) in enumerate(Gout.edges(data=True)):
if 'geometry' in attr_dict.keys():
attr_dict['geometry'] = attr_dict['geometry'].wkt
G0 = ox.simplify_graph(Gout.to_directed())
G0 = G0.to_undirected()
# reprojecting graph screws up lat lon, so convert to string?
# Gout = ox.project_graph(G0) # broken
Gout = G0
if verbose:
node = list(Gout.nodes())[-1]
print(node, "random node props:", Gout.nodes[node])
# print an edge
edge_tmp = list(Gout.edges())[-1]
print(edge_tmp, "random edge props:",
Gout.get_edge_data(edge_tmp[0], edge_tmp[1]))
# When the simplify funciton combines edges, it concats multiple
# edge properties into a list. This means that 'geometry_pix' is now
# a list of geoms. Convert this to a linestring with
# shaply.ops.linemergeconcats
# BUG, GOOF, ERROR IN OSMNX PROJECT, SO NEED TO MANUALLY SET X, Y FOR NODES!!??
if manually_reproject_nodes:
# make sure geometry is utm for nodes?
for i, (n, attr_dict) in enumerate(Gout.nodes(data=True)):
attr_dict['x'] = attr_dict['utm_east']
attr_dict['y'] = attr_dict['utm_north']
if verbose:
print("Merge 'geometry' linestrings...")
keys_tmp = [
'geometry_wkt', 'geometry_pix', 'geometry_latlon_wkt',
'geometry_utm_wkt'
]
for key_tmp in keys_tmp:
if verbose:
print("Merge", key_tmp, "...")
for i, (u, v, attr_dict) in enumerate(Gout.edges(data=True)):
if key_tmp not in attr_dict.keys():
continue
geom = attr_dict[key_tmp]
if type(geom) == list:
# check if the list items are wkt strings, if so, create
# linestrigs
if (type(geom[0]) == str):
geom = [shapely.wkt.loads(ztmp) for ztmp in geom]
# merge geoms
geom_out = shapely.ops.linemerge(geom)
elif type(geom) == str:
geom_out = shapely.wkt.loads(geom)
else:
geom_out = geom
# now straighten edge with rdp
if rdp_epsilon > 0:
coords = list(geom_out.coords)
new_coords = rdp.rdp(coords, epsilon=rdp_epsilon)
geom_out_rdp = LineString(new_coords)
geom_out_final = geom_out_rdp
else:
geom_out_final = geom_out
len_out = geom_out_final.length
# updata edge properties
attr_dict[key_tmp] = geom_out_final
# update length
if key_tmp == 'geometry_pix':
attr_dict['length_pix'] = len_out
if key_tmp == 'geometry_utm_wkt':
attr_dict['length_utm'] = len_out
# assign 'geometry' tag to geometry_wkt
# !! assign 'geometry' tag to geometry_utm_wkt
key_tmp = 'geometry_wkt' # 'geometry_utm_wkt'
for i, (u, v, attr_dict) in enumerate(Gout.edges(data=True)):
line = attr_dict['geometry_utm_wkt']
if type(line) == str:
attr_dict['geometry'] = shapely.wkt.loads(line)
else:
attr_dict['geometry'] = attr_dict[key_tmp]
attr_dict['geometry_wkt'] = attr_dict['geometry'].wkt
# set length
attr_dict['length'] = attr_dict['geometry'].length
# update wkt_pix?
attr_dict['wkt_pix'] = attr_dict['geometry_pix'].wkt
# update 'length_pix'
attr_dict['length_pix'] = np.sum([attr_dict['length_pix']])
# print a random node and edge
if verbose:
node_tmp = list(Gout.nodes())[-1]
print(node_tmp, "random node props:", Gout.nodes[node_tmp])
# print an edge
edge_tmp = list(Gout.edges())[-1]
print("random edge props for edge:", edge_tmp, " = ",
Gout.edges[edge_tmp[0], edge_tmp[1], 0])
# get a few stats (and set to graph properties)
Gout.graph['N_nodes'] = len(Gout.nodes())
Gout.graph['N_edges'] = len(Gout.edges())
# get total length of edges
tot_meters = 0
for i, (u, v, attr_dict) in enumerate(Gout.edges(data=True)):
tot_meters += attr_dict['length']
if verbose:
print("Length of edges (km):", tot_meters / 1000)
Gout.graph['Tot_edge_km'] = tot_meters / 1000
if verbose:
print("G.graph:", Gout.graph)
# save graph
nx.write_gpickle(Gout, out_file, protocol=pickle_protocol)
def wkt_to_G(wkt_list,
im_file=None,
min_subgraph_length_pix=30,
simplify_graph=True,
verbose=False):
if verbose:
print("Running wkt_list_to_nodes_edges()...")
node_loc_dic, edge_dic = wkt_list_to_nodes_edges(wkt_list)
if verbose:
print("Creating G...")
G0 = nodes_edges_to_G(node_loc_dic, edge_dic)
if verbose:
print(" len(G.nodes():", len(G0.nodes()))
print(" len(G.edges():", len(G0.edges()))
if verbose:
print("Clean out short subgraphs")
G0 = clean_sub_graphs(G0,
min_length=min_subgraph_length_pix,
max_nodes_to_skip=30,
weight='length_pix',
verbose=False,
super_verbose=False)
if len(G0) == 0:
return G0
# geo coords
if im_file:
if verbose:
print("Running get_node_geo_coords()...")
G1 = get_node_geo_coords(G0, im_file, verbose=verbose)
if verbose:
print("Running get_edge_geo_coords()...")
G1 = get_edge_geo_coords(G1, im_file, verbose=verbose)
if verbose:
print("projecting graph...")
G_projected = ox.project_graph(G1)
Gout = G_projected #G_simp
else:
Gout = G0
if simplify_graph:
if verbose:
print("Simplifying graph")
G0 = ox.simplify_graph(Gout.to_directed())
G0 = G0.to_undirected()
Gout = ox.project_graph(G0)
if verbose:
print("Merge 'geometry' linestrings...")
keys_tmp = ['geometry_pix', 'geometry_latlon_wkt', 'geometry_utm_wkt']
for key_tmp in keys_tmp:
if verbose:
print("Merge", key_tmp, "...")
for i, (u, v, attr_dict) in enumerate(Gout.edges(data=True)):
if (i % 10000) == 0:
if verbose:
print(i, u, v)
geom = attr_dict[key_tmp]
#print (i, u, v, "geom:", geom)
#print (" type(geom):", type(geom))
if type(geom) == list:
# check if the list items are wkt strings, if so, create
# linestrigs
if (type(geom[0]) == str
): # or (type(geom_pix[0]) == unicode):
geom = [shapely.wkt.loads(ztmp) for ztmp in geom]
# merge geoms
#geom = shapely.ops.linemerge(geom)
#attr_dict[key_tmp] = geom
attr_dict[key_tmp] = shapely.ops.linemerge(geom)
elif type(geom) == str:
attr_dict[key_tmp] = shapely.wkt.loads(geom)
else:
pass
# assign 'geometry' tag to geometry_utm_wkt
for i, (u, v, attr_dict) in enumerate(Gout.edges(data=True)):
if verbose:
print("Create 'geometry' field in edges...")
#geom_pix = attr_dict[key_tmp]
line = attr_dict['geometry_utm_wkt']
if type(line) == str: # or type(line) == unicode:
attr_dict['geometry'] = shapely.wkt.loads(line)
else:
attr_dict['geometry'] = attr_dict['geometry_utm_wkt']
# update wkt_pix?
#print ("attr_dict['geometry_pix':", attr_dict['geometry_pix'])
attr_dict['wkt_pix'] = attr_dict['geometry_pix'].wkt
# update 'length_pix'
attr_dict['length_pix'] = np.sum([attr_dict['length_pix']])
Gout = ox.project_graph(Gout)
if verbose:
# get a few stats (and set to graph properties)
print("Number of nodes: {}".format(len(Gout.nodes())))
print("Number of edges: {}".format(len(Gout.edges())))
#print ("Number of nodes:", len(Gout.nodes()))
#print ("Number of edges:", len(Gout.edges()))
Gout.graph['N_nodes'] = len(Gout.nodes())
Gout.graph['N_edges'] = len(Gout.edges())
# get total length of edges
tot_meters = 0
for i, (u, v, attr_dict) in enumerate(Gout.edges(data=True)):
tot_meters += attr_dict['length']
if verbose:
print("Length of edges (km):", tot_meters / 1000)
Gout.graph['Tot_edge_km'] = tot_meters / 1000
if verbose:
print("G.graph:", Gout.graph)
return Gout
import scipy as sp
if False:
print("-------------graph-style-examples-----------------")
print(trajB.shape)
trajM = trajB.pivot_table(index="origin", columns="destination", values="n", aggfunc=np.sum)
setR = trajM.index.isin(trajM.columns)
setC = trajM.columns.isin(trajM.index)
trajM = trajM.loc[setR, setC]
plt.imshow(trajM.values)
plt.show()
trajS = sp.sparse.coo_matrix(trajM, dtype=np.int8)
G = nx.Graph(trajS)
simp_G = ox.simplify_graph(G)
val_map = {'A': 1.0, 'D': 0.5714285714285714, 'H': 0.0}
values = [val_map.get(node, 0.45) for node in G.nodes()]
edge_labels = dict([((u, v,), d['weight']) for u, v, d in G.edges(data=True)])
red_edges = [('C', 'D'), ('D', 'A')]
edge_colors = ['black' if not edge in red_edges else 'red' for edge in G.edges()]
pos = nx.spring_layout(G)
nx.draw_networkx_edge_labels(G, pos, edge_labels=edge_labels)
nx.draw(G, pos, node_color=values, node_size=1500, edge_color=edge_colors, edge_cmap=plt.cm.Reds)
plt.show()
G = nx.florentine_families_graph()
adjacency_matrix = nx.adjacency_matrix(G)
G = nx.fast_gnp_random_graph(100, 0.04)
adj_matrix = nx.adjacency_matrix(G)
def analyze_city(boundary, crs, local_edges):
"""Analyze correspondence between Local and OSM bikeways throughout a city.
Parameters
----------
boundary : :class:`shapely.geometry.Polygon`
City boundary projected in WGS 84
crs : epsg coordinate system
Local coordinate system in meters (e.g., UTM 10: {'init': 'epsg:26910'})
local_edges : :class:`geopandas.GeoDataFrame`
Output from `structure_bikeways_shapefile`
Returns
-------
:obj:`tuple`
* :class:`pandas.DataFrame`
Output from `summarize_bikeway_correspondance`
* :class:`geopandas.GeoDataFrame`
OSM edges with OSM and local bikeway data attached
"""
# Define OSM tag filter
# Importantly, no paths with 'highway':'service' or 'highway':'motorway' tags will be returned
tag_filter = (
'["area"!~"yes"]["highway"!~"service|footway|motor|proposed|construction|abandoned|platform|raceway"]'
'["bicycle"!~"no"]["access"!~"private"]')
# Download the OSM data
overpass_jsons = ox.osm_net_download(boundary, custom_filter=tag_filter)
overpass_json = sp.merge_overpass_jsons(overpass_jsons)
# Define bikeway columns and associated labels
bikeway_types = [
'off_street_path', 'bike_blvd', 'separated_bike_lane', 'bike_lane',
'shoulder', 'sharrow', 'bike_route'
]
# Parse Overpass JSON into bikeway types
overpass_parsed = sp.parse_osm_tags(overpass_json,
bikeway_types,
true_value=1,
false_value=0,
none_value=np.nan)
# Specify attributes to include in graph
path_tags = (bikeway_types + ['highway'])
ox.config(useful_tags_path=path_tags)
# Convert json to graph
G = ox.create_graph([overpass_parsed])
# Simply graph by removing all nodes that are not intersections or dead ends
G = ox.simplify_graph(G, strict=True)
# Make graph undirected
G = nx.to_undirected(G)
# Convert graph to geodataframes
osm_edges = ox.graph_to_gdfs(G, nodes=False)
# Project to local coordinate system
osm_edges = osm_edges.to_crs(crs)
# Project city boundary to local coordinate system
boundary, _ = ox.project_geometry(boundary, to_crs=crs)
# Constrain edges to those intersecting the city boundary polygon
osm_edges = sp.gdf_intersecting_polygon(osm_edges, boundary)
# Summarize bikeway values stored in lists
osm_edges[bikeway_types] = osm_edges[bikeway_types].applymap(
lambda x: sp.nan_any(x, 1, np.nan))
# Idenfity largest available highway type
def largest_highway(highways):
# Specify highway order,
# largest (least bikable) to smallest (most bikable)
highway_order = [
'trunk', 'primary', 'secondary', 'tertiary', 'unclassified',
'residential', 'living_street', 'cycleway'
]
highways = sp.listify(highways)
# Strip '_link' from tags
highways = [x[:-5] if x[-5:] == '_link' else x for x in highways]
# If list includes one of these tags, return the biggest one
ranked_highways = [x for x in highways if x in highway_order]
if len(ranked_highways) > 0:
ranks = [highway_order.index(x) for x in ranked_highways]
return highway_order[min(ranks)]
# Otherwise, return 'other'
else:
return 'other'
osm_edges['highway'] = osm_edges['highway'].apply(largest_highway)
# Restrict edges to bikeable highway types
bikable = [
'primary', 'secondary', 'tertiary', 'unclassified', 'residential',
'living_street', 'cycleway'
]
osm_edges = osm_edges[osm_edges['highway'].isin(bikable)].copy()
# Project local edges to local coordinate system
local_edges = local_edges.to_crs(crs)
# Restrict to local edges intersecting the city boundary
local_edges = sp.gdf_intersecting_polygon(local_edges, boundary)
# Match local edges to OSM edges
analysis_columns = bikeway_types + ['geometry']
# Match dataframes
osm_matches = sp.match_lines_by_hausdorff(
sp.select_columns(
osm_edges, analysis_columns,
suffix='_osm').rename(columns={'geometry_osm': 'geometry'}),
sp.select_columns(
local_edges, analysis_columns,
suffix='_local').rename(columns={'geometry_local': 'geometry'}),
constrain_target_features=True,
distance_tolerance=20,
azimuth_tolerance=20,
match_fields=True)
# Identify local and osm bikeway columns
joint_bikeway_cols = [
column for column in osm_matches.columns
if any(bikeway in column for bikeway in bikeway_types)
]
# Reduce lists to a single single binary value
osm_matches[joint_bikeway_cols] = osm_matches[joint_bikeway_cols].applymap(
lambda x: sp.nan_any(x, 1, np.nan))
# Drop records without a bikeway in either dataset
osm_matches = osm_matches.dropna(how='all', subset=joint_bikeway_cols)
# Reclassify NaN values as 0
osm_matches = osm_matches.fillna(0)
# Function fo calculate composite bikeways
def composite_columns(matches, columns, suffix):
# Select relevent columns
relevent_columns = sp.select_columns(matches,
[x + suffix for x in columns])
# Assess whether there are any values of 1 across each row
return relevent_columns.apply(lambda x: sp.nan_any(x, 1, 0), axis=1)
# Define exclusive and shared bikeway types
exclusive_bikeways = ['bike_lane', 'separated_bike_lane']
shared_bikeways = ['bike_blvd', 'sharrow', 'bike_route']
# Calculate composite of exclusive bikeways
osm_matches['exclusive_bikeway_osm'] = composite_columns(
osm_matches, exclusive_bikeways, '_osm')
osm_matches['exclusive_bikeway_local'] = composite_columns(
osm_matches, exclusive_bikeways, '_local')
# Calculate composite of shared bikeways
osm_matches['shared_bikeway_osm'] = composite_columns(
osm_matches, shared_bikeways, '_osm')
osm_matches['shared_bikeway_local'] = composite_columns(
osm_matches, shared_bikeways, '_local')
# Calculate composite of all bikeways
osm_matches['any_bikeway_osm'] = composite_columns(osm_matches,
bikeway_types, '_osm')
osm_matches['any_bikeway_local'] = composite_columns(
osm_matches, bikeway_types, '_local')
# Calculate the length of each edge
osm_matches['length'] = osm_matches['geometry'].apply(lambda x: x.length)
# Add labels to bikeway types
bikeway_labels = [
'Off Street Path', 'Bike Boulevard', 'Separated Bike Lane',
'Bike Lane', 'Shoulder', 'Sharrow', 'Bike Route'
]
bikeway_labels = OrderedDict(zip(bikeway_types, bikeway_labels))
# Add labels for composite bikeway types
bikeway_labels.update({'exclusive_bikeway': 'Exclusive'})
bikeway_labels.update({'shared_bikeway': 'Shared'})
bikeway_labels.update({'any_bikeway': 'Any'})
# Calculate summaries
summaries = summarize_bikeway_correspondance(osm_matches, bikeway_labels)
return summaries, osm_matches
def simple_get_roads(config):
"""
Use osmnx to get a simplified version of open street maps for the city
Writes osm_nodes and osm_ways shapefiles to MAP_FP
Args:
city
Returns:
None, but creates the following shape files:
osm_ways.shp - the simplified road network
osm_nodes.shp - the intersections and dead ends
And creates the following directory:
all_nodes - containing edges and nodes directories
for the unsimplified road network
"""
# confirm if a polygon is available for this city, which determines which
# graph function is appropriate
print("searching nominatim for " + str(config['city']) + " polygon")
polygon_pos = find_osm_polygon(config['city'])
if (polygon_pos is not None):
print("city polygon found in OpenStreetMaps at position " +
str(polygon_pos) + ", building graph of roads within " +
"specified bounds")
G1 = ox.graph_from_place(config['city'],
network_type='drive',
simplify=False,
which_result=polygon_pos)
else:
# City & lat+lng+radius required from config to graph from point
if ('city' not in list(config.keys()) or config['city'] is None):
sys.exit('city is required in config file')
if ('city_latitude' not in list(config.keys())
or config['city_latitude'] is None):
sys.exit('city_latitude is required in config file')
if ('city_longitude' not in list(config.keys())
or config['city_longitude'] is None):
sys.exit('city_longitude is required in config file')
if ('city_radius' not in list(config.keys())
or config['city_radius'] is None):
sys.exit('city_radius is required in config file')
print("no city polygon found in OpenStreetMaps, building graph of " +
"roads within " + str(config['city_radius']) + "km of city " +
str(config['city_latitude']) + " / " +
str(config['city_longitude']))
G1 = ox.graph_from_point(
(config['city_latitude'], config['city_longitude']),
distance=config['city_radius'] * 1000,
network_type='drive',
simplify=False)
G = ox.simplify_graph(G1)
# Label endpoints
streets_per_node = ox.count_streets_per_node(G)
for node, count in list(streets_per_node.items()):
if count <= 1:
G.nodes()[node]['dead_end'] = True
# osmnx creates a directory for the nodes and edges
# Store all nodes, since they can be other features
ox.save_graph_shapefile(G1, filename='all_nodes', folder=MAP_FP)
# Store simplified network
ox.save_graph_shapefile(G, filename='temp', folder=MAP_FP)
# Copy and remove temp directory
tempdir = os.path.join(MAP_FP, 'temp')
for filename in os.listdir(os.path.join(tempdir, 'edges')):
name, extension = filename.split('.')
shutil.move(os.path.join(tempdir, 'edges', filename),
os.path.join(MAP_FP, 'osm_ways.' + extension))
for filename in os.listdir(os.path.join(tempdir, 'nodes')):
name, extension = filename.split('.')
shutil.move(os.path.join(tempdir, 'nodes', filename),
os.path.join(MAP_FP, 'osm_nodes.' + extension))
shutil.rmtree(tempdir)
def wkt_to_G(params):
'''Execute all functions'''
n_threads_max = 12
wkt_list, im_file, min_subgraph_length_pix, \
node_iter, edge_iter, \
min_spur_length_m, simplify_graph, \
rdp_epsilon,\
manually_reproject_nodes, \
out_file, graph_dir, n_threads, verbose \
= params
print("im_file:", im_file)
# print("wkt_list:", wkt_list)
pickle_protocol = 4
t0 = time.time()
if verbose:
print ("Running wkt_list_to_nodes_edges()...")
node_loc_dic, edge_dic = wkt_list_to_nodes_edges(wkt_list,
node_iter=node_iter,
edge_iter=edge_iter)
t1 = time.time()
if verbose:
print ("Time to run wkt_list_to_nodes_egdes():", t1 - t0, "seconds")
#print ("node_loc_dic:", node_loc_dic)
#print ("edge_dic:", edge_dic)
if verbose:
print ("Creating G...")
G0 = nodes_edges_to_G(node_loc_dic, edge_dic)
if verbose:
print (" len(G.nodes():", len(G0.nodes()))
print (" len(G.edges():", len(G0.edges()))
#for edge_tmp in G0.edges():
# print ("\n 0 wtk_to_G():", edge_tmp, G0.edge[edge_tmp[0]][edge_tmp[1]])
t2 = time.time()
if verbose:
print ("Time to run nodes_edges_to_G():", t2-t1, "seconds")
# This graph will have a unique edge for each line segment, meaning that
# many nodes will have degree 2 and be in the middle of a long edge.
# run clean_sub_graph() in 04_skeletonize.py? - Nope, do it here
# so that adding small terminals works better...
if verbose:
print ("Clean out short subgraphs")
G1 = clean_sub_graphs(G0, min_length=min_subgraph_length_pix,
weight='length_pix', verbose=verbose,
super_verbose=False)
t3 = time.time()
if verbose:
print ("Time to run clean_sub_graphs():", t3-t2, "seconds")
t3 = time.time()
# G1 = G0
if len(G1) == 0:
return G1
# print ("Simplifying graph")
# G0 = ox.simplify_graph(G0.to_directed())
# G0 = G0.to_undirected()
# #G0 = ox.project_graph(G0)
# #G_p_init = create_edge_linestrings(G_p_init, remove_redundant=True, verbose=False)
# t3 = time.time()
# print (" len(G.nodes():", len(G0.nodes()))
# print (" len(G.edges():", len(G0.edges()))
# print ("Time to run simplify graph:", t30 - t3, "seconds")
#for edge_tmp in G0.edges():
# print ("\n 1 wtk_to_G():", edge_tmp, G0.edge[edge_tmp[0]][edge_tmp[1]])
#edge_tmp = G0.edges()[5]
#print (edge_tmp, "G0.edge props:", G0.edge[edge_tmp[0]][edge_tmp[1]])
# geo coords
if im_file:
if verbose:
print ("Running get_node_geo_coords()...")
# let's not over multi-thread a multi-thread
if n_threads > 1:
n_threads_tmp = 1
else:
n_threads_tmp = n_threads_max
G1 = get_node_geo_coords(G1, im_file, n_threads=n_threads_tmp,
verbose=verbose)
t4= time.time()
if verbose:
print ("Time to run get_node_geo_coords():", t4-t3, "seconds")
if verbose:
print ("Running get_edge_geo_coords()...")
# let's not over multi-thread a multi-thread
if n_threads > 1:
n_threads_tmp = 1
else:
n_threads_tmp = n_threads_max
G1 = get_edge_geo_coords(G1, im_file, n_threads=n_threads_tmp,
verbose=verbose)
t5 = time.time()
if verbose:
print ("Time to run get_edge_geo_coords():", t5-t4, "seconds")
if verbose:
print("pre projection...")
node = list(G1.nodes())[-1]
if verbose:
print(node, "random node props:", G1.nodes[node])
# print an edge
edge_tmp = list(G1.edges())[-1]
print(edge_tmp, "random edge props:", G1.get_edge_data(edge_tmp[0], edge_tmp[1]))
if verbose:
print ("projecting graph...")
G_projected = ox.project_graph(G1)
# get geom wkt (for printing/viewing purposes)
for i,(u,v,attr_dict) in enumerate(G_projected.edges(data=True)):
attr_dict['geometry_wkt'] = attr_dict['geometry'].wkt
if verbose:
print("post projection...")
node = list(G_projected.nodes())[-1]
print(node, "random node props:", G_projected.nodes[node])
# print an edge
edge_tmp = list(G_projected.edges())[-1]
print(edge_tmp, "random edge props:", G_projected.get_edge_data(edge_tmp[0], edge_tmp[1]))
t6 = time.time()
if verbose:
print ("Time to project graph:", t6-t5, "seconds")
# simplify
#G_simp = ox.simplify_graph(G_projected.to_directed())
#ox.plot_graph(G_projected)
#G1.edge[19][22]
Gout = G_projected #G_simp
else:
Gout = G0
# ###########################################################################
# # remove short edges?
# # this is done in 04_skeletonize.remove_small_terminal()
# t31 = time.time()
# Gout = remove_short_edges(Gout, min_spur_length_m=min_spur_length_m)
# t32 = time.time()
# print("Time to remove_short_edges():", t32 - t31, "seconds")
# ###########################################################################
if simplify_graph:
if verbose:
print("Simplifying graph")
t7 = time.time()
# 'geometry' tag breaks simplify, so maket it a wkt
for i,(u,v,attr_dict) in enumerate(G_projected.edges(data=True)):
if 'geometry' in attr_dict.keys():
attr_dict['geometry'] = attr_dict['geometry'].wkt
G0 = ox.simplify_graph(Gout.to_directed())
G0 = G0.to_undirected()
# print("G0")
# node = list(G0.nodes())[-1]
# print(node, "random node props:", G0.nodes[node])
#Gout = G0
# reprojecting graph screws up lat lon, so convert to string?
Gout = ox.project_graph(G0)
# print("Gout")
# node = list(Gout.nodes())[-1]
# print(node, "random node props:", Gout.nodes[node])
if verbose:
print("post simplify...")
node = list(Gout.nodes())[-1]
print(node, "random node props:", Gout.nodes[node])
# print an edge
edge_tmp = list(Gout.edges())[-1]
print(edge_tmp, "random edge props:", Gout.get_edge_data(edge_tmp[0], edge_tmp[1]))
t8 = time.time()
if verbose:
print("Time to run simplify graph:", t8-t7, "seconds")
# When the simplify funciton combines edges, it concats multiple
# edge properties into a list. This means that 'geometry_pix' is now
# a list of geoms. Convert this to a linestring with
# shaply.ops.linemergeconcats
# BUG, GOOF, ERROR IN OSMNX PROJECT, SO NEED TO MANUALLY SET X, Y FOR NODES!!??
if manually_reproject_nodes:
# make sure geometry is utm for nodes?
for i, (n, attr_dict) in enumerate(Gout.nodes(data=True)):
attr_dict['x'] = attr_dict['utm_east']
attr_dict['y'] = attr_dict['utm_north']
# if simplify_graph:
# print ("Simplifying graph")
# t7 = time.time()
# G0 = ox.simplify_graph(Gout.to_directed())
# Gout = G0.to_undirected()
# #Gout = ox.project_graph(G0)
# t8 = time.time()
# print ("Time to run simplify graph:", t8-t7, "seconds")
# # When the simplify funciton combines edges, it concats multiple
# # edge properties into a list. This means that 'geometry_pix' is now
# # a list of geoms. Convert this to a linestring with
# # shaply.ops.linemergeconcats
if verbose:
print ("Merge 'geometry' linestrings...")
keys_tmp = ['geometry_wkt', 'geometry_pix', 'geometry_latlon_wkt',
'geometry_utm_wkt']
for key_tmp in keys_tmp:
if verbose:
print ("Merge", key_tmp, "...")
for i,(u,v,attr_dict) in enumerate(Gout.edges(data=True)):
if key_tmp not in attr_dict.keys():
continue
if (i % 10000) == 0:
print (i, u , v)
geom = attr_dict[key_tmp]
#print (i, u, v, "geom:", geom)
#print (" type(geom):", type(geom))
if type(geom) == list:
# check if the list items are wkt strings, if so, create
# linestrigs
if (type(geom[0]) == str):# or (type(geom_pix[0]) == unicode):
geom = [shapely.wkt.loads(ztmp) for ztmp in geom]
# merge geoms
#geom = shapely.ops.linemerge(geom)
#attr_dict[key_tmp] = geom
geom_out = shapely.ops.linemerge(geom)
# attr_dict[key_tmp] = shapely.ops.linemerge(geom)
elif type(geom) == str:
geom_out = shapely.wkt.loads(geom)
# attr_dict[key_tmp] = shapely.wkt.loads(geom)
else:
geom_out = geom
# now straighten edge with rdp
if rdp_epsilon > 0:
if verbose and ((i % 10000) == 0):
print(" Applying rdp...")
coords = list(geom_out.coords)
new_coords = rdp.rdp(coords, epsilon=rdp_epsilon)
geom_out_rdp = LineString(new_coords)
geom_out_final = geom_out_rdp
else:
geom_out_final = geom_out
len_out = geom_out_final.length
# updata edge properties
attr_dict[key_tmp] = geom_out_final
# update length
if key_tmp == 'geometry_pix':
attr_dict['length_pix'] = len_out
if key_tmp == 'geometry_utm_wkt':
attr_dict['length_utm'] = len_out
# assign 'geometry' tag to geometry_wkt
# !! assign 'geometry' tag to geometry_utm_wkt
key_tmp = 'geometry_wkt' # 'geometry_utm_wkt'
for i,(u,v,attr_dict) in enumerate(Gout.edges(data=True)):
if verbose and ((i % 10000) == 0):
print ("Create 'geometry' field in edges...")
line = attr_dict['geometry_utm_wkt']
if type(line) == str:# or type(line) == unicode:
attr_dict['geometry'] = shapely.wkt.loads(line)
else:
attr_dict['geometry'] = attr_dict[key_tmp]
attr_dict['geometry_wkt'] = attr_dict['geometry'].wkt
# set length
attr_dict['length'] = attr_dict['geometry'].length
# update wkt_pix?
#print ("attr_dict['geometry_pix':", attr_dict['geometry_pix'])
attr_dict['wkt_pix'] = attr_dict['geometry_pix'].wkt
# update 'length_pix'
attr_dict['length_pix'] = np.sum([attr_dict['length_pix']])
# Gout = ox.project_graph(Gout)
# print a random node and edge
if verbose:
node_tmp = list(Gout.nodes())[-1]
print(node_tmp, "random node props:", Gout.nodes[node_tmp])
# print an edge
edge_tmp = list(Gout.edges())[-1]
print("random edge props for edge:", edge_tmp, " = ",
Gout.edges[edge_tmp[0], edge_tmp[1], 0])
# get a few stats (and set to graph properties)
if verbose:
logger1.info("Number of nodes: {}".format(len(Gout.nodes())))
logger1.info("Number of edges: {}".format(len(Gout.edges())))
#print ("Number of nodes:", len(Gout.nodes()))
#print ("Number of edges:", len(Gout.edges()))
Gout.graph['N_nodes'] = len(Gout.nodes())
Gout.graph['N_edges'] = len(Gout.edges())
# get total length of edges
tot_meters = 0
for i,(u,v,attr_dict) in enumerate(Gout.edges(data=True)):
tot_meters += attr_dict['length']
if verbose:
print ("Length of edges (km):", tot_meters/1000)
Gout.graph['Tot_edge_km'] = tot_meters/1000
if verbose:
print ("G.graph:", Gout.graph)
# save
if len(Gout.nodes()) == 0:
nx.write_gpickle(Gout, out_file, protocol=pickle_protocol)
return
# # print a node
# node = list(Gout.nodes())[-1]
# print (node, "random node props:", Gout.nodes[node])
# # print an edge
# edge_tmp = list(Gout.edges())[-1]
# #print (edge_tmp, "random edge props:", G.edges([edge_tmp[0], edge_tmp[1]])) #G.edge[edge_tmp[0]][edge_tmp[1]])
# print (edge_tmp, "random edge props:", Gout.get_edge_data(edge_tmp[0], edge_tmp[1]))
# save graph
if verbose:
logger1.info("Saving graph to directory: {}".format(graph_dir))
#print ("Saving graph to directory:", graph_dir)
nx.write_gpickle(Gout, out_file, protocol=pickle_protocol)
# # save shapefile as well?
# if save_shapefiles:
# logger1.info("Saving shapefile to directory: {}".format(graph_dir))
# try:
# ox.save_graph_shapefile(G, filename=image_id.split('.')[0] , folder=graph_dir, encoding='utf-8')
# except:
# print("Cannot save shapefile...")
# #out_file2 = os.path.join(graph_dir, image_id.split('.')[0] + '.graphml')
# #ox.save_graphml(G, image_id.split('.')[0] + '.graphml', folder=graph_dir)
#
# # plot, if desired
# if make_plots:
# print ("Plotting graph...")
# outfile_plot = os.path.join(graph_dir, image_id)
# print ("outfile_plot:", outfile_plot)
# ox.plot_graph(G, fig_height=9, fig_width=9,
# #save=True, filename=outfile_plot, margin=0.01)
# )
# #plt.tight_layout()
# plt.savefig(outfile_plot, dpi=400)
t7 = time.time()
if verbose:
print ("Total time to run wkt_to_G():", t7-t0, "seconds")
return # Gout
def simple_link_mapper(graph, strict=True):
"""
CREDIT: https://github.com/gboeing/osmnx/blob/master/osmnx/simplify.py
------- modified for different functionality -------
The default graph extracted has lots of links due to every bend in the road
being classified as a link. Due to graph having lots of links (and the map matcher requiring it to do so),
we need a way to map these smaller links to their simplified ones so we can
model the longer roads as desired. Helper function.
Returns a dictionary unsimplified edges to their corresponding simplified links
as well as the proportion (lengths) of those simplified links they represent.
Parameters:
----------
graph (networkx multidigraph)
strict (bool): if False, allow nodes to be end points even if they fail all other rules
but have edges with different OSM IDs
Returns:
-------
link_mapper (dictionary): Dictionary that maps unsimplified edges to their corresponding simplified links
as well as the proportion (lengths) of those simplified links they represent.
"""
# first identify all the nodes that are endpoints
endpoints = set([
node for node in graph.nodes()
if ox.is_endpoint(graph, node, strict=strict)
])
paths_to_simplify = []
# for each endpoint node, look at each of its successor nodes
for node in endpoints:
for successor in graph.successors(node):
if successor not in endpoints:
# if the successor is not an endpoint, build a path from the
# endpoint node to the next endpoint node
try:
path = ox.build_path(graph,
successor,
endpoints,
path=[node, successor])
paths_to_simplify.append(path)
except RuntimeError:
continue
else:
# just add node, successor into paths to simplify anyways (so it get's added to link_mapper later)
paths_to_simplify.append([node, successor])
# Now we have a list of paths to simplify
# we will now use the edges of a pre-simplified graph for comparison
simplified_graph = ox.simplify_graph(graph, strict=True)
simple_edge_data = return_edge_data(simplified_graph)
#also get the edge data for normal graph
edge_data = return_edge_data(graph)
link_mapper = {}
# now iterate through the paths to simplify
for path in paths_to_simplify:
# simplified edge corresponds to first and last of path
simple_edge = get_edge_name(path[0], path[-1])
simple_edge_length = simple_edge_data[simple_edge]['length']
# now iterate through interstitial nodes
for u, v in zip(path[:-1], path[1:]):
edge_name_1 = get_edge_name(u, v)
# now take into account other direction too
edge_name_2 = get_edge_name(v, u)
# now find what proportion of simple link length this link is
proportion = edge_data[edge_name_1]['length'] / simple_edge_length
link_mapper[edge_name_1] = {
'simple_link': simple_edge,
'proportion': proportion
}
link_mapper[edge_name_2] = {
'simple_link': simple_edge,
'proportion': proportion
}
return link_mapper
network_type='bike')
ox.plot_graph(G2,
node_zorder=2,
node_size=10,
node_alpha=1,
node_color='r',
bgcolor='w',
edge_linewidth=0.2,
use_geom=True,
axis_off=False,
show=False,
close=False)
ox.graph_to_gdfs(G2, nodes=False)
G2_proj = ox.project_graph(G2)
G3 = ox.consolidate_intersections(G2_proj, tolerance=10, rebuild_graph=True)
G4 = ox.simplify_graph(G3.copy())
ox.plot_graph(G3,
node_zorder=2,
node_size=10,
node_alpha=1,
node_color='r',
bgcolor='w',
edge_linewidth=0.2,
use_geom=True,
axis_off=False,
show=False,
close=False)
gdf_edge_sample = ox.graph_to_gdfs(G3, nodes=False)
gdf_edge_sample.groupby(['u', 'v']).count()
def network_dfs(self):
g = self.graph_latlon
if not self.strict:
g = ox.simplify_graph(g, strict=self.strict)
g = ox.add_edge_bearings(g, precision=1)
n, e = ox.graph_to_gdfs(g)
e = e.reset_index() # Method graph_to_gdfs changed to multiindex df
network_edges, network_nodes_small = dbl_cleaning(ndf=n, edf=e)
network_edges = network_edges.join(network_nodes_small, on='u')
network_edges = network_edges.rename(columns={
'u': 'n1',
'y': 'lat1',
'x': 'lon1'
})
network_edges = network_edges.join(network_nodes_small, on='v')
network_edges = network_edges.rename(columns={
'v': 'n2',
'y': 'lat2',
'x': 'lon2'
})
x1, y1 = zip(
*project_point(list(zip(network_edges.lon1, network_edges.lat1))))
x2, y2 = zip(
*project_point(list(zip(network_edges.lon2, network_edges.lat2))))
network_edges = network_edges.assign(x1=x1, y1=y1, x2=x2, y2=y2)
network_edges['edge'] = list(
zip(network_edges['n1'].values, network_edges['n2'].values))
network_edges.reset_index(
inplace=True
) # From hereon the unique index of an edge is just its position in df
network_edges = network_edges.rename(columns={'index': '_id'})
self.graph_latlon = g
self.graph_xy = ox.project_graph(self.graph_latlon)
self.network_edges = network_edges
self._get_network_nodes(network_edges)
# link node_tags to specific edge, osmid not unique over edges after simplification
nearest = ox.get_nearest_edges(self.graph_xy,
self.node_tags.x.to_list(),
self.node_tags.y.to_list(),
method='kdtree',
dist=1)
n1, n2, _ = zip(*nearest)
test_b1 = network_edges[['_id', 'edge',
'bearing']][network_edges.edge.isin(
list(zip(n1, n2)))].values
test_b2 = network_edges[['_id', 'edge',
'bearing']][network_edges.edge.isin(
list(zip(n2, n1)))].values
self.node_tags['edge'] = [ij for ij in zip(n1, n2)]
self.node_tags.reset_index(inplace=True)
self.node_tags = self.node_tags.merge(
self.network_edges[['edge', 'bearing']],
on='edge',
suffixes=('', '_edge'))
diff_b = abs(self.node_tags['bearing'] -
self.node_tags['bearing_edge'])
for i, j in diff_b.iteritems():
if (j > 45) and not self.node_tags.junction[i]:
self.node_tags.at[i,
'edge'] = (self.node_tags.at[i, 'edge'][1],
self.node_tags.at[i, 'edge'][0])
self.node_tags.drop('bearing_edge', axis=1, inplace=True)
self.node_tags = self.node_tags.merge(
self.network_edges[['_id', 'edge', 'bearing']],
on='edge',
suffixes=('', '_edge'))
diff_b2 = abs(self.node_tags['bearing'] -
self.node_tags['bearing_edge'])
breakpoint()
# check if nearest edge is in right direction, problem with two way streets
self.node_tags.set_index('index', inplace=True)
self.node_tags.sort_index(inplace=True)
def pedestrians_first(boundaries,
id_code,
name,
folder_name='',
buffer_dist=100,#m
headway_threshold=10,#min
to_test = [
'healthcare',
'schools',
'h+s',
'libraries',
'carfree',
'blocks',
'density',
'transit',
'pnb',
#'special',
],
distances = { #network buffers, in meters
'healthcare': 1000,
'schools': 1000,
'libraries': 1000,
'transit': 500,
'special': 500,
'pnb': 250,
},
overpass = False,
patch_length = 5, #km
boundary_buffer = 0, #km
blocks_simplification = 15, #m
gtfs_files = [],
):
dt = datetime.datetime.now()
logger = logging.getLogger()
logger.setLevel(logging.CRITICAL)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
fh = logging.FileHandler('log_filename.txt')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
logger.addHandler(fh)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
logger.addHandler(ch)
useful_tags = ox.settings.useful_tags_way + ['cycleway', 'cycleway:left', 'cycleway:right']
ox.config(use_cache=True, log_console=True, useful_tags_way=useful_tags)
if folder_name != '' and not folder_name[-1:] == '/':
folder_name += '/'
if boundary_buffer > 0:
bound_latlon = gpd.GeoDataFrame(geometry = [boundaries])
bound_latlon.crs = {'init':'epsg:4326'}
longitude = round(numpy.mean(bound_latlon.geometry.centroid.x),10)
utm_zone = int(math.floor((longitude + 180) / 6) + 1)
utm_crs = '+proj=utm +zone={} +ellps=WGS84 +datum=WGS84 +units=m +no_defs'.format(utm_zone)
bound_utm = bound_latlon.to_crs(utm_crs)
bound_utm.geometry = bound_utm.geometry.buffer(boundary_buffer*1000)
bound_latlon = bound_utm.to_crs(epsg=4326)
boundaries = bound_latlon.geometry.unary_union
bbox = boundaries.bounds
crs = None
if type(boundaries) == shapely.geometry.multipolygon.MultiPolygon:
patches = []
for poly in list(boundaries):
patches += make_patches(poly, patch_length=patch_length)
else:
patches = make_patches(boundaries, patch_length=patch_length)
longitude_factor = 0.00898 # degrees per km
longitude_factor_m = 0.00898 / 1000 # degrees per m
latitude_factor = (math.cos(abs(boundaries.bounds[1])*0.0174533))/111.319
latitude_factor_m = latitude_factor / 1000
print('Evaluating Pedestrians First indicators in',name)
print('Measuring',str(to_test))
quilt_isochrone_polys = {}
quilt_center_nodes = {}
for service in to_test:
quilt_isochrone_polys[service] = False
quilt_center_nodes[service] = []
patch_times = []
all_coords={}
testing_services = []
for service in ['healthcare', 'schools', 'libraries']:
if service in to_test:
testing_services.append(service)
#all_coords[service] = get_point_locations(boundaries, queries[service])
if len(testing_services) > 0:
handler = get_service_locations.ServiceHandler()
handler.apply_file(folder_name+'/city.o5m', locations=True)
for service in testing_services:
all_coords[service] = handler.locationlist[service]
citywide_carfree = handler.carfreelist
#should I finish this?
if 'highways' in to_test:
highway_filter = '["area"!~"yes"]["highway"~"motorway|trunk"]'
G_hwys = ox.graph_from_polygon(boundaries,
custom_filter=highway_filter,
retain_all=True)
G_hwys = ox.project_graph(G_hwys)
edges_hwys = ox.utils_graph.graph_to_gdfs(G_hwys, nodes=False)
edges_polyline = edges_hwys.geometry.unary_union
increment_dist = 25
distances = np.arange(0, edges_polyline.length, increment_dist)
points = [edges_polyline.interpolate(distance) for distance in distances] + [edges_polyline.boundary[1]]
multipoint = unary_union(points)
#need to switch back to latlon and put in all_coords['highway']
if 'special' in to_test:
if os.path.isfile(folder_name+'/special.shp'):
testing_services.append('special')
special = gpd.read_file(folder_name+'/special.shp')
all_coords['special'] = [(pt.y, pt.x) for pt in special.geometry]
if 'transit' in to_test:
testing_services.append('transit')
#this approach does not let us combine OpenMobilityData files with local files.
if len(gtfs_files) == 0:
sources = gtfs_parser.get_feed_infos(gtfs_parser.get_relevant_locs(bbox))
#note! This doesn't reflect a buffered boundary if boundary_buffer > 0
transit_stop_sets = gtfs_parser.count_all_sources(sources,
source_type = 'openmobilitydata',
headwaylim = headway_threshold * 2)
else:
transit_stop_sets = gtfs_parser.count_all_sources(gtfs_files,
source_type = 'local_files',
headwaylim = headway_threshold * 2)
if 'pnb' in to_test:
testing_services.append('pnb')
if len(to_test) > 0 and to_test != ["blocks"]:
for p_idx, patch in enumerate(patches):
try:
patch_start = datetime.datetime.now()
unbuffered_patch = patch
max_service_dist_km = max(distances.values())/1000
patch = shapely.geometry.box(
patch.bounds[0] - (max_service_dist_km * longitude_factor),
patch.bounds[1] - (max_service_dist_km * latitude_factor),
patch.bounds[2] + (max_service_dist_km * longitude_factor),
patch.bounds[3] + (max_service_dist_km * latitude_factor)
)
walk_filter = ('["area"!~"yes"]["highway"!~"link|motor'
'|proposed|construction|abandoned'
'|platform|raceway"]'
'["service"!~"parking_aisle|driveway"]'
'["foot"!~"no"]["service"!~"private"]'
'{}').format(ox.settings.default_access)
if overpass:
G = ox.graph_from_polygon(patch,
custom_filter=walk_filter,
simplify=False,
retain_all=True)
else:
boundingarg = '-b='
boundingarg += str(patch.bounds[0])+','
boundingarg += str(patch.bounds[1])+','
boundingarg += str(patch.bounds[2])+','
boundingarg += str(patch.bounds[3])
subprocess.check_call(['osmconvert',
str(folder_name)+'/citywalk.o5m',
boundingarg,
#'--complete-ways',
'--drop-broken-refs',
'-o=patch.osm'])
G = ox.graph_from_xml('patch.osm',
simplify=False, retain_all=True)
os.remove('patch.osm')
if 'pnb' in to_test:
subprocess.check_call(['osmconvert',
str(folder_name)+'/cityhighways.o5m',
boundingarg,
#'--complete-ways',
'--drop-broken-refs',
'-o=allhwyspatch.osm'])
G_allhwys = ox.graph_from_xml('allhwyspatch.osm',
simplify=False, retain_all=True)
os.remove('allhwyspatch.osm')
G.remove_nodes_from(list(nx.isolates(G)))
G_allhwys.remove_nodes_from(list(nx.isolates(G_allhwys)))
simple_G = ox.simplify_graph(G)
center_nodes = {}
for service in all_coords.keys():
if service in ['healthcare','schools','libraries','special']:
center_nodes[service] = []
for coord in all_coords[service]:
lat = coord[0]
lon = coord[1]
if unbuffered_patch.bounds[0] < lon < unbuffered_patch.bounds[2] and unbuffered_patch.bounds[1] < lat < unbuffered_patch.bounds[3]:
point = shapely.geometry.Point(lon,lat)
nearest = ox.get_nearest_node(simple_G, coord)
if not nearest in center_nodes[service]:
center_nodes[service].append(nearest)
if 'pnb' in to_test:
center_nodes['pnb'] = []
allhwys_gdf = ox.graph_to_gdfs(G_allhwys, nodes=False)
print(allhwys_gdf.columns)
waytypes = []
if 'highway' in allhwys_gdf.columns:
waytypes.append(allhwys_gdf[allhwys_gdf['highway'] == 'cycleway'])
if 'cycleway' in allhwys_gdf.columns:
waytypes.append(allhwys_gdf[allhwys_gdf['cycleway'] == 'track'])
if 'cycleway:left' in allhwys_gdf.columns:
waytypes.append(allhwys_gdf[allhwys_gdf['cycleway:left'] == 'track'])
if 'cycleway:right' in allhwys_gdf.columns:
waytypes.append(allhwys_gdf[allhwys_gdf['cycleway:right'] == 'track'])
cycletracknodes = []
for waytype in waytypes:
for edge in waytype.index:
if not edge[0] in cycletracknodes:
cycletracknodes.append(edge[0])
if not edge[1] in cycletracknodes:
cycletracknodes.append(edge[1])
for node in cycletracknodes:
if node in simple_G.nodes:
center_nodes['pnb'].append(node)
if 'transit' in to_test:
center_nodes['transit'] = []
transit_centers = {}
for service_idx, service in enumerate(transit_stop_sets):
for stop_id in service.keys():
lat = float(service[stop_id][1])
lon = float(service[stop_id][2])
headway = float(service[stop_id][0])
if unbuffered_patch.bounds[0] < lon < unbuffered_patch.bounds[2] and unbuffered_patch.bounds[1] < lat < unbuffered_patch.bounds[3]:
center_node = ox.get_nearest_node(simple_G, (lat, lon))
if center_node not in transit_centers:
transit_centers[center_node] = {service_idx : headway} #store the headway value
elif service_idx not in transit_centers[center_node].keys():
transit_centers[center_node][service_idx] = headway
elif headway < transit_centers[center_node][service_idx]:
transit_centers[center_node][service_idx] = headway
for center_node in transit_centers.keys():
if len(transit_centers[center_node]) > 0:
inv_headway = 0
min_hw = 100
for service_idx in transit_centers[center_node].keys():
inv_headway += 1 / transit_centers[center_node][service_idx]
if transit_centers[center_node][service_idx] < min_hw:
min_hw = transit_centers[center_node][service_idx]
headway = 1 / inv_headway
if headway <= headway_threshold:
center_nodes['transit'].append(center_node)
# Project Graph
if not crs: #this only runs once, crs is invariable over patches
G = ox.project_graph(G)
crs = G.graph['crs']
else:
G = ox.project_graph(G, to_crs=crs)
G = ox.simplify_graph(G)
isochrone_polys = {}
failures = {}
for service in to_test:
failures[service] = 0
# Get polygons
for service in testing_services:
isochrone_polys[service], fails = isochrones.make_iso_polys(G, center_nodes[service], distance=distances[service], edge_buff=buffer_dist)
failures[service] += fails
for service in isochrone_polys.keys():
if service not in quilt_isochrone_polys.keys() or not quilt_isochrone_polys[service]:
quilt_isochrone_polys[service] = isochrone_polys[service]
elif isochrone_polys[service]:
quilt_isochrone_polys[service] = shapely.ops.cascaded_union([quilt_isochrone_polys[service],isochrone_polys[service]])
for service in center_nodes.keys():
quilt_center_nodes[service] = quilt_center_nodes[service] + center_nodes[service]
patch_time = datetime.datetime.now() - patch_start
print("finished patch #",p_idx,'out of', len(patches),"in",str(patch_time))
patch_times.append(patch_time)
except ox._errors.EmptyOverpassResponse:
pass
except ValueError:
pass #sorry
#except:
# print("GOT SOME ERROR FOR PATCH", p_idx)
# except ValueError:
# print('ValueError')
# now = str(datetime.datetime.now())
# with open('error'+now+'.txt','w') as errout:
# traceback.print_exc(limit=3,file=errout)
# print('saved to error'+now+'.txt')
#epsg = 32600+int(crs.split(' ')[1].split('=')[1]) #This is wild --
#osmnx seems to just give all data in northern-hemisphere format
#Sorry about the stupid parsing of the projection definition, I'm lazy
results = {'name':name,'id_code':id_code}
if not os.path.exists(folder_name):
os.makedirs(folder_name)
boundaries_latlon = gpd.GeoDataFrame(geometry=[boundaries])
boundaries_latlon.crs = {'init':'epsg:4326'}
try:
boundaries_utm = boundaries_latlon.to_crs(crs)
except ValueError:
longitude = round(numpy.mean(boundaries_latlon.geometry.centroid.x),10)
utm_zone = int(math.floor((longitude + 180) / 6) + 1)
utm_crs = '+proj=utm +zone={} +ellps=WGS84 +datum=WGS84 +units=m +no_defs'.format(utm_zone)
crs = utm_crs
boundaries_utm = boundaries_latlon.to_crs(crs)
stats = rasterstats.zonal_stats(boundaries_latlon, 'pop_dens.tif', stats=['sum'])
total_pop = stats[0]['sum']
for service in testing_services:
if quilt_isochrone_polys[service]:
service_utm = gpd.GeoDataFrame(geometry = [quilt_isochrone_polys[service]])
service_utm.crs = crs#{'init':'epsg:'+str(epsg)}
service_utm.geometry = service_utm.geometry.simplify(15) #maybe this should be after the population calculation
service_utm = gpd.overlay(service_utm ,boundaries_utm, how='intersection')
service_latlon = service_utm.to_crs(epsg=4326)
service_latlon.to_file(folder_name+service+'latlon'+'.geojson', driver='GeoJSON')
stats = rasterstats.zonal_stats(service_latlon, 'pop_dens.tif', stats=['sum'])
total_PNS = stats[0]['sum']
print("\n")
print('Total People Near Service for', service, ":", total_PNS)
print(100*total_PNS/total_pop,"% of",total_pop)
results[service] = total_PNS / total_pop
else:
print ('NO SERVICE FOR', service)
results[service] = 0
if 'carfree' in to_test:
print("getting carfree")
if citywide_carfree:
carfree_latlon = gpd.GeoDataFrame(geometry = citywide_carfree)
#just a latlon list of points
carfree_latlon.crs = {'init':'epsg:4326'}
carfree_utm = carfree_latlon.to_crs(crs)
carfree_utm.geometry = carfree_utm.geometry.buffer(100)
#this is the analysis, the 100m buffer
carfree_utm = gpd.GeoDataFrame(geometry = [shapely.ops.cascaded_union(carfree_utm.geometry)])
carfree_utm.geometry = carfree_utm.geometry.simplify(10)
carfree_utm = gpd.overlay(carfree_utm ,boundaries_utm, how='intersection')
carfree_utm.crs = crs
carfree_latlon = carfree_utm.to_crs('epsg:4326')
stats = rasterstats.zonal_stats(carfree_latlon, 'pop_dens.tif', stats=['sum'])
total_carfree = stats[0]['sum']
print("\n")
print('Total People Near Service for carfree', ":", total_carfree)
print(100*total_carfree/total_pop,"% of",total_pop)
results['carfree'] = total_carfree / total_pop
carfree_latlon = carfree_utm.to_crs('epsg:4326')
carfree_latlon.to_file(folder_name+'carfreelatlon'+'.geojson', driver='GeoJSON')
else:
print ('NO SERVICE FOR carfree')
results['carfree'] = 0
if 'h+s' in to_test:
if quilt_isochrone_polys['healthcare'] and quilt_isochrone_polys['schools']:
service = 'h+s'
intersect = quilt_isochrone_polys['healthcare'].intersection(quilt_isochrone_polys['schools'])
if type(intersect) == shapely.geometry.collection.GeometryCollection:
intersect = [obj for obj in intersect if type(obj) == shapely.geometry.polygon.Polygon]
intersect = shapely.geometry.MultiPolygon(intersect)
hs_utm = gpd.GeoDataFrame(geometry = [intersect])
hs_utm.crs = crs#{'init':'epsg:'+str(epsg)} #maybe this should be after the population calculation
if hs_utm.geometry.area.sum() != 0:
hs_utm = gpd.overlay(hs_utm ,boundaries_utm, how='intersection')
hs_utm.geometry = hs_utm.geometry.simplify(15)
hs_latlon = hs_utm.to_crs(epsg=4326)
hs_latlon.to_file(folder_name+service+'latlon'+'.geojson', driver='GeoJSON')
stats = rasterstats.zonal_stats(hs_latlon, 'pop_dens.tif', stats=['sum'])
total_PNS = stats[0]['sum']
print("\n")
print('Total People Near Service for', service, ":", total_PNS)
print(100*total_PNS/total_pop,"% of",total_pop)
results[service] = total_PNS / total_pop
else:
print ('NO SERVICE FOR h+s')
results['h+s'] = 0
else:
print ('NO SERVICE FOR h+s')
results['h+s'] = 0
if 'density' in to_test:
density = rasterstats.zonal_stats(boundaries,
'pop_dens.tif',
stats = [],
add_stats={'weighted': weighted_pop_density}
)[0]['weighted']
results['density'] = density / 0.0625 #km^2 / pixel
print('weighted pop density', results['density'])
with rasterio.open('pop_dens.tif') as dataset:
out_image, out_transform = rasterio.mask.mask(dataset, [boundaries], crop=True)
out_meta = dataset.meta
out_meta.update({"driver": "GTiff",
"height": out_image.shape[1],
"width": out_image.shape[2],
"transform": out_transform})
with rasterio.open(folder_name+"pop_dens.tif", "w", **out_meta) as dest:
dest.write(out_image)
#garbage collection
quilt_isochrone_polys = None
isochrone_polys = None
out_image = None
dest = None
dataset = None
a = None
b = None
import gc
gc.collect()
if 'blocks' in to_test:
print("getting blocks")
patches = make_patches(boundaries, patch_length=patch_length)
print ("cut", len(list(patches)),"patches for block size in",name)
outblocks = []
block_counts = []
for n, patch in enumerate(patches):
print("patch"+str(n)+" of "+str(len(patches)) )
unbuffered_patch = gpd.GeoSeries(patch, crs={'init':'epsg:4326'})
unbuffered_patch = unbuffered_patch.to_crs(crs)[0]
buffer_dist = 1
patch = shapely.geometry.box(
patch.bounds[0] - (buffer_dist * longitude_factor),
patch.bounds[1] - (buffer_dist * latitude_factor),
patch.bounds[2] + (buffer_dist * longitude_factor),
patch.bounds[3] + (buffer_dist * latitude_factor)
)
if overpass:
G = ox.graph_from_polygon(patch, custom_filter=walk_filter, simplify=False, retain_all=True)
else:
boundingarg = '-b='
boundingarg += str(patch.bounds[0])+','
boundingarg += str(patch.bounds[1])+','
boundingarg += str(patch.bounds[2])+','
boundingarg += str(patch.bounds[3])
subprocess.check_call(['osmconvert',
folder_name+'/citywalk.o5m',
boundingarg,
#'--complete-ways',
'--drop-broken-refs',
'-o=patch.osm'])
try:
G = ox.graph_from_xml('patch.osm', simplify=False, retain_all=True)
except:
G = False
if G:
try:
G = ox.project_graph(G, to_crs=crs)
G = ox.simplify_graph(G)
streets = ox.utils_graph.graph_to_gdfs(G, nodes = False)
if not streets.empty:
streets = shapely.geometry.MultiLineString(list(streets.geometry))
merged = shapely.ops.linemerge(streets)
if merged:
borders = shapely.ops.unary_union(merged)
blocks = list(shapely.ops.polygonize(borders))
all_blocks = []
selected_areas = []
for block in blocks:
if 500 < block.area: #< 200000000:
if block.interiors:
block = shapely.geometry.Polygon(block.exterior)
if block.centroid.within(unbuffered_patch):
area = round(block.area, 3)
perim = round(block.length, 3)
lemgth = round((perim * perim) / area, 3)
if blocks_simplification:
block = block.simplify(blocks_simplification)
all_blocks.append((block, area, perim, lemgth))
if (lemgth < 50) and (1000 < area < 1000000):
selected_areas.append(area)
outblocks += all_blocks
block_counts.append(len(all_blocks))
else:
block_counts.append(0)
print('not merged!')
else:
block_counts.append(0)
except:
print('Hawassa Error')
block_counts.append(0)
else:
block_counts.append(0)
#export
patch_densities = gpd.GeoDataFrame(geometry = list(patches))
patch_densities['block_count'] = block_counts
patch_densities.crs = {'init':'epsg:4326'}
patch_densities_utm = patch_densities.to_crs(crs)
patch_densities_utm['density'] = patch_densities_utm.block_count / (patch_densities_utm.area /1000000)
patch_densities_latlon = patch_densities_utm.to_crs(epsg=4326)
patch_densities_latlon.to_file(folder_name+'patch_densities'+'latlon'+'.geojson', driver='GeoJSON')
a = gpd.GeoDataFrame(geometry=[block[0] for block in outblocks])
a.crs = crs#{'init':'epsg:'+str(epsg)}
a['area'] = [block[1] for block in outblocks]
a['perim'] = [block[2] for block in outblocks]
a['lemgth'] = [block[3] for block in outblocks]
a['density'] = [1000000/block[1] for block in outblocks]
b = a.to_crs(epsg=4326)
b.to_file(folder_name+'blocks'+'latlon'+'.geojson', driver='GeoJSON')
#b.to_file(folder_name+'blocks'+'latlon'+'.shp')
filtered_blocks = []
for block in outblocks:
if block[3] < 50:
if 1000 < block[1] < 1000000:
filtered_blocks.append(block)
c = gpd.GeoDataFrame(geometry=[block[0] for block in outblocks])
c.crs = crs#{'init':'epsg:'+str(epsg)}
c['area'] = [block[1] for block in outblocks]
c['perim'] = [block[2] for block in outblocks]
c['lemgth'] = [block[3] for block in outblocks]
try:
blockmedian = statistics.median([block[1] for block in filtered_blocks])
print('median block density')
results['blockmedian_density'] = 1000000 / blockmedian
print(results['blockmedian_density'])
except:
results['blockmedian_density'] = 0
print('BLOCK MEDIAN COULD NOT BE CALCULATED')
try:
blockmean = statistics.mean([block[1] for block in filtered_blocks])
print('mean block density')
results['blockmean_density'] = 1000000 / blockmean
print(results['blockmean_density'])
except:
results['blockmean_density'] = 0
print('BLOCK MEAN COULD NOT BE CALCULATED')
ft = datetime.datetime.now()
print("total", str(ft-dt))
results['calctime'] = str(ft-dt)
results['total_pop'] = total_pop
with open(folder_name+"results.json","w") as output:
output.write(json.dumps(results))
for file in ['city.o5m','cityhighways.o5m','citywalk.o5m']:
if os.path.exists(folder_name+'/'+file):
os.remove(folder_name+'/'+file)
return results
