def test_graph_save_load():
# save graph as shapefile and geopackage
G = ox.graph_from_place(place1, network_type="drive")
ox.save_graph_shapefile(G)
ox.save_graph_geopackage(G, directed=False)
# save/load geopackage and convert graph to/from node/edge GeoDataFrames
fp = ".temp/data/graph-dir.gpkg"
ox.save_graph_geopackage(G, filepath=fp, directed=True)
gdf_nodes1 = gpd.read_file(fp, layer="nodes").set_index("osmid")
gdf_edges1 = gpd.read_file(fp, layer="edges").set_index(["u", "v", "key"])
G2 = ox.graph_from_gdfs(gdf_nodes1, gdf_edges1)
G2 = ox.graph_from_gdfs(gdf_nodes1, gdf_edges1, graph_attrs=G.graph)
gdf_nodes2, gdf_edges2 = ox.graph_to_gdfs(G2)
assert set(gdf_nodes1.index) == set(gdf_nodes2.index) == set(G.nodes) == set(G2.nodes)
assert set(gdf_edges1.index) == set(gdf_edges2.index) == set(G.edges) == set(G2.edges)
# create random boolean graph/node/edge attributes
attr_name = "test_bool"
G.graph[attr_name] = False
bools = np.random.randint(0, 2, len(G.nodes))
node_attrs = {n: bool(b) for n, b in zip(G.nodes, bools)}
nx.set_node_attributes(G, node_attrs, attr_name)
bools = np.random.randint(0, 2, len(G.edges))
edge_attrs = {n: bool(b) for n, b in zip(G.edges, bools)}
nx.set_edge_attributes(G, edge_attrs, attr_name)
# save/load graph as graphml file
ox.save_graphml(G, gephi=True)
ox.save_graphml(G, gephi=False)
filepath = Path(ox.settings.data_folder) / "graph.graphml"
G2 = ox.load_graphml(
filepath,
graph_dtypes={attr_name: ox.io._convert_bool_string},
node_dtypes={attr_name: ox.io._convert_bool_string},
edge_dtypes={attr_name: ox.io._convert_bool_string},
)
# verify everything in G is equivalent in G2
assert tuple(G.graph.keys()) == tuple(G2.graph.keys())
assert tuple(G.graph.values()) == tuple(G2.graph.values())
z = zip(G.nodes(data=True), G2.nodes(data=True))
for (n1, d1), (n2, d2) in z:
assert n1 == n2
assert tuple(d1.keys()) == tuple(d2.keys())
assert tuple(d1.values()) == tuple(d2.values())
z = zip(G.edges(keys=True, data=True), G2.edges(keys=True, data=True))
for (u1, v1, k1, d1), (u2, v2, k2, d2) in z:
assert u1 == u2
assert v1 == v2
assert k1 == k2
assert tuple(d1.keys()) == tuple(d2.keys())
assert tuple(d1.values()) == tuple(d2.values())
# test custom data types
nd = {"osmid": str}
ed = {"length": str, "osmid": float}
G2 = ox.load_graphml(filepath, node_dtypes=nd, edge_dtypes=ed)
def test_find_nearest():
# get graph
G = ox.graph_from_point(location_point, dist=500, network_type="drive")
# convert graph to node/edge GeoDataFrames and back again
gdf_nodes, gdf_edges = ox.graph_to_gdfs(
G, nodes=True, edges=True, node_geometry=True, fill_edge_geometry=True
)
assert len(gdf_nodes) == len(G)
assert len(gdf_edges) == len(G.edges(keys=True))
G = ox.graph_from_gdfs(gdf_nodes, gdf_edges)
assert len(gdf_nodes) == len(G)
assert len(gdf_edges) == len(G.edges(keys=True))
# get nearest node
nn, d = ox.get_nearest_node(G, location_point, method="euclidean", return_dist=True)
# get nearest nodes: haversine, kdtree, balltree
X = gdf_nodes["x"].head()
Y = gdf_nodes["y"].head()
nn1 = ox.get_nearest_nodes(G, X, Y)
nn2 = ox.get_nearest_nodes(G, X, Y, method="kdtree")
nn3 = ox.get_nearest_nodes(G, X, Y, method="balltree")
# get nearest edge
u, v, k, g, d = ox.get_nearest_edge(G, location_point, return_geom=True, return_dist=True)
# get nearest edges: haversine, kdtree, balltree
ne1 = ox.get_nearest_edges(G, X, Y)
ne2 = ox.get_nearest_edges(G, X, Y, method="kdtree")
ne3 = ox.get_nearest_edges(G, X, Y, method="balltree", dist=0.0001)
def load_drive_graph(path='db/shenzhen20170701-all.osm', ):
"""
load original open street map data of a city and extract drivable road network
:param path: original osm xml format data
:return: drivable graph
"""
shenzhen_all = ox.graph_from_xml(path)
sz_nodes, sz_edges = ox.graph_to_gdfs(shenzhen_all,
fill_edge_geometry=True)
not_valid_highway = 'cycleway|footway|path|pedestrian|steps|track|corridor|elevator' \
'|escalator|proposed|construction|bridleway|abandoned|platform' \
'|raceway|service'.split('|')
not_valid_service = 'parking|parking_aisle|driveway|private|emergency_access'.split(
'|')
print('original_edges', sz_edges.shape)
sz_gdfs = sz_edges.loc[((sz_edges['highway'].notna())
& (sz_edges['area'] != 'yes')
& (sz_edges['access'] != 'private')
& (~sz_edges['service'].isin(not_valid_service)))]
for tag in not_valid_highway:
sz_gdfs = sz_gdfs.loc[sz_gdfs['highway'] != tag].copy(deep=True)
print('drivable_edges:', sz_gdfs.shape)
shenzhen_drive = ox.graph_from_gdfs(sz_nodes, sz_gdfs)
return shenzhen_drive
def plot_community_bus_routes(G):
nodes, edges = ox.graph_to_gdfs(G)
# get nodes that are on the routes
# and create a graph containing only those
route_nodes = nodes[(nodes["community_route"])]
route_nodes_graph = ox.graph_from_gdfs(route_nodes, edges)
# convert community labels to integers so that get_node_colors_by_attr
# can use the community attribute
for x, y in route_nodes_graph.nodes(data=True):
if "community" in y:
y["community"] = int(y["community"])
node_colours = ox.plot.get_node_colors_by_attr(route_nodes_graph,
attr="community",
cmap="tab20")
# graph_from_gdfs creates empty nodes so need
# to update node_colours to include those so
# that plot graph function will work correctly
other_nodes = {
x: (0, 0, 0, 0)
for x, y in G.nodes(data=True) if x not in node_colours.index
}
series = pd.Series(other_nodes)
node_colours = node_colours.append(series)
ox.plot_graph(route_nodes_graph,
node_color=node_colours,
edge_color="w",
node_size=15)
def generateHeatMaps(self, df):
for index, row in df.iterrows():
print("")
print("LOCATION ROW ::::::::::::::::::::::::::::: #", index)
print("")
if not any(d['placeid'] == row["placeid"]
for d in self.visitedLocations):
tempRow = row
tempRow["visited"] = 0.0
self.visitedLocations.append(tempRow)
vi = self.find_index(self.visitedLocations, "placeid",
row["placeid"])
self.visitedLocations[vi]["visited"] += self.visitPoints
location_point = (row["lat"], row["lon"])
distance = self.viewDistance
G = self.getMap(location_point, distance)
#G = ox.graph_from_point(location_point, network_type='drive', dist=distance, simplify=False)
self.timeStepAll(G)
#Make geodataframes from graph data
nodes, edges = ox.graph_to_gdfs(G, nodes=True, edges=True)
#Now make the same graph, but this time from the geodataframes
#This will help retain the 'visits' columns
nodes['visits'] = self.assignVisitsToNodes(nodes, G)
nodes['visitsS'] = self.mS * nodes['visits']
#print(self.visitedLocations)
#G = ox.save_load.gdfs_to_graph(nodes, edges)
G = ox.graph_from_gdfs(nodes, edges)
#fig, ax =ox.plot_graph(G,show=True, close=False)
#ox.plot_graph(G)
if index > 1:
#ox.plot_graph(G,fig_height=8,fig_width=8,node_size=nodes['visits'], node_color=nc)
#nc = ox.plot.get_node_colors_by_attr(G,'visits',cmap='plasma')
#nc = ox.plot.get_node_colors_by_attr(G, 'visits',start=0.0,stop=self.visitedMaxPoints, cmap='plasma')
print("nodes['visits']",
nodes[nodes.visits > 0.00].to_string())
nc = ox.plot.get_node_colors_by_attr(G,
'visits',
cmap='plasma')
fig, ax = ox.plot_graph(G,
node_color=nc,
node_size=nodes['visitsS'],
edge_color='#333333',
bgcolor='k',
show=False)
fig.savefig('data/MAPS/test_' + str(index) + '.png')
def dbl_cleaning(ndf, edf):
if 'busway:left' and 'busway:right' not in edf.columns:
network_edges = edf.loc[:, [
'u', 'v', 'oneway', 'osmid', 'highway', 'length', 'bearing',
'geometry', 'lanes'
]]
network_nodes_small = ndf.loc[:, ['y', 'x']]
return network_edges, network_nodes_small
new_rows = []
left_na = pd.isna(edf['busway:left'])
right_na = pd.isna(edf['busway:right'])
edf = edf.assign(dbl_left=~left_na)
edf = edf.assign(dbl_right=~right_na)
# Temporal addition to change all dbl in network
for r, v in edf.iterrows():
if v.busway == 'opposite_lane' and not v.dbl_left:
edf.loc[r, 'dbl_left'] = True
edf = edf.drop(['busway:left', 'busway:right'], axis=1)
dbl_bool = np.logical_and(edf['dbl_left'].values, edf['oneway'].values)
gdf_val = edf[['u', 'v', 'bearing']].values
new_index = len(edf)
for row, val in edf.iterrows():
if dbl_bool[row]:
new_row = val.copy()
new_row['u'] = int(gdf_val[row][1])
new_row['v'] = int(gdf_val[row][0])
new_row['lanes'] = 1
new_row['bearing'] = gdf_val[row][2] - 180
new_row['osmid'] = -1
new_row['geometry'] = [
LineString([
ndf['geometry'][gdf_val[row][1]],
ndf['geometry'][gdf_val[row][0]]
])
]
new_row = gpd.GeoDataFrame(dict(new_row), index=[new_index])
new_index += 1
new_rows.append(new_row)
if new_rows:
new_rows = pd.concat(new_rows, axis=0)
edf = pd.concat([edf, new_rows], axis=0)
edf.set_index(['u', 'v', 'key'], inplace=True)
new_graph = ox.graph_from_gdfs(ndf, edf)
ndf, edf = ox.graph_to_gdfs(new_graph)
edf.reset_index(inplace=True)
network_edges = edf.loc[:, [
'u', 'v', 'oneway', 'osmid', 'highway', 'length', 'bearing',
'geometry', 'lanes', 'dbl_left', 'dbl_right'
]]
network_nodes_small = ndf.loc[:, ['y', 'x']]
return network_edges, network_nodes_small
def test_graph_save_load():
# save graph as shapefile and geopackage
G = ox.graph_from_place(place1, network_type="drive")
ox.save_graph_shapefile(G)
ox.save_graph_geopackage(G, directed=False)
# save/load geopackage and convert graph to/from node/edge GeoDataFrames
fp = ".temp/data/graph-dir.gpkg"
ox.save_graph_geopackage(G, filepath=fp, directed=True)
gdf_nodes1 = gpd.read_file(fp, layer="nodes").set_index("osmid")
gdf_edges1 = gpd.read_file(fp, layer="edges").set_index(["u", "v", "key"])
G2 = ox.graph_from_gdfs(gdf_nodes1, gdf_edges1, graph_attrs=G.graph)
gdf_nodes2, gdf_edges2 = ox.graph_to_gdfs(G2)
assert set(gdf_nodes1.index) == set(gdf_nodes2.index) == set(G.nodes) == set(G2.nodes)
assert set(gdf_edges1.index) == set(gdf_edges2.index) == set(G.edges) == set(G2.edges)
# save/load graph as graphml file
ox.save_graphml(G, gephi=True)
ox.save_graphml(G, gephi=False)
filepath = Path(ox.settings.data_folder) / "graph.graphml"
G2 = ox.load_graphml(filepath)
# verify everything in G is equivalent in G2
for (n1, d1), (n2, d2) in zip(G.nodes(data=True), G2.nodes(data=True)):
assert n1 == n2
assert d1 == d2
for (u1, v1, k1, d1), (u2, v2, k2, d2) in zip(
G.edges(keys=True, data=True), G2.edges(keys=True, data=True)
):
assert u1 == u2
assert v1 == v2
assert k1 == k2
assert tuple(d1.keys()) == tuple(d2.keys())
assert tuple(d1.values()) == tuple(d2.values())
for (k1, v1), (k2, v2) in zip(G.graph.items(), G2.graph.items()):
assert k1 == k2
assert v1 == v2
# test custom data types
nd = {"osmid": str}
ed = {"length": str, "osmid": float}
G2 = ox.load_graphml(filepath, node_dtypes=nd, edge_dtypes=ed)
def prepare_graph(self, g):
"""Takes in a graph, add speed and travel times, updates travel times per edge
Input:
graph
Output:
graph
"""
new_g = ox.add_edge_speeds(g)
new_g = ox.add_edge_bearings(new_g)
edges_g = ox.graph_to_gdfs(new_g, nodes=False) # Graph edges DF
nodes_g = ox.graph_to_gdfs(new_g, edges=False) # Graph nodes DF
edges_g_updated = self.update_speed_in_edges(
edges_g) # change the speed for calamity roads
edges_g_updated['travel_time2'] = edges_g_updated.apply(
lambda x: self.travel_time_recalc(x), axis=1)
new_graph = ox.graph_from_gdfs(
nodes_g, edges_g_updated) ## Putting the DF back to the graph
return new_graph, edges_g_updated
def set_risk_to_graph(G, risk_path):
"""
Append risk attributes to the nodes and the edges of a graph.
Parameters
----------
G : networkx.MultiDiGraph
input graph
risk_path : string or pathlib.Path
path to the DataFrame with the crash information file including ext.
Returns
-------
G : networkx.MultiDiGraph
"""
# TODO use DF crashes to computes the risk
nodes_proj, edges_proj = ox.graph_to_gdfs(G, nodes=True, edges=True)
edges_proj["risk"] = edges_proj["length"] % 1
G = ox.graph_from_gdfs(nodes_proj, edges_proj)
return G
import networkx as nx
import osmnx as ox
ox.config(use_cache=True, log_console=True)
# download street network data from OSM and construct a MultiDiGraph model
G = ox.graph_from_point((37.79, -122.41), dist=750, network_type="drive")
# impute edge (driving) speeds and calculate edge traversal times
G = ox.add_edge_speeds(G)
G = ox.add_edge_travel_times(G)
# you can convert MultiDiGraph to/from geopandas GeoDataFrames
gdf_nodes, gdf_edges = ox.graph_to_gdfs(G)
G = ox.graph_from_gdfs(gdf_nodes, gdf_edges, graph_attrs=G.graph)
# convert MultiDiGraph to DiGraph to use nx.betweenness_centrality function
# choose between parallel edges by minimizing travel_time attribute value
D = ox.utils_graph.get_digraph(G, weight="travel_time")
# calculate node betweenness centrality, weighted by travel time
bc = nx.betweenness_centrality(D, weight="travel_time", normalized=True)
nx.set_node_attributes(G, values=bc, name="bc")
# plot the graph, coloring nodes by betweenness centrality
nc = ox.plot.get_node_colors_by_attr(G, "bc", cmap="plasma")
fig, ax = ox.plot_graph(G,
bgcolor="k",
node_color=nc,
node_size=50,
def get_graph(gdf_nodes, gdf_edges):
#Returns: nodes and edges from pickle
#Cached by Streamlit
return ox.graph_from_gdfs(gdf_nodes, gdf_edges)
sidewalks['osmid'] = sidewalks.index.map(lambda x: 100000 * x)
with open('test/sw_points_dict.pkl', 'wb') as pklfile:
pkl.dump(sw_points, pklfile)
assert sidewalks.crs == LOCAL_CRS
assert sw_points.crs == LOCAL_CRS
sidewalks = sidewalks.to_crs(GLOBAL_CRS)
sw_points = sw_points.to_crs(GLOBAL_CRS)
sw_points['x'] = sw_points.geometry.map(lambda x: x.coords[0][1])
sw_points['y'] = sw_points.geometry.map(lambda x: x.coords[0][0])
sidewalks_G = ox.graph_from_gdfs(sw_points, sidewalks)
def angle_reverse(G):
rev_edges = nx.reverse(G).edges(data=True)
def reverse_line(linestring):
p0, p1 = linestring.coords[:]
return LineString([Point(p1), Point(p0)])
def rev_angle(dic):
dic['angle_deg'] = -dic['angle_deg']
dic['geometry'] = reverse_line(dic['geometry'])
return dic
return [(u, v, rev_angle(dat)) for (u, v, dat) in rev_edges]
sidewalks_G.add_edges_from(angle_reverse(sidewalks_G))
# set up for model
traffic_x = df_traffic_grouped_with_features_train.drop(['u','v','key','aadb','osmid','geometry'],axis=1)
traffic_y = df_traffic_grouped_with_features_train['aadb']
# Traffic Model
regr = RandomForestRegressor(max_depth=10, random_state=0)
regr.fit(traffic_x,traffic_y)
np.array(traffic_y)/regr.predict(traffic_x.sort_index(axis=1))
## Apply model to edges in network
df_undirected_edges_with_features['setyear'] = 2018
traffic_as_input = df_undirected_edges_with_features.drop(['u','v','key','osmid','geometry'],axis=1).sort_index(axis=1)
df_edges_undirected['aadb_predictions'] = regr.predict(traffic_as_input)
G_undirected_with_traffic_weights = ox.graph_from_gdfs(df_nodes,df_edges_undirected)
## uncomment for traffic model EDA
ec = ox.plot.get_edge_colors_by_attr(G_undirected_with_traffic_weights, 'aadb_predictions', cmap='plasma',num_bins=20)
fig,ax = ox.plot_graph(G_undirected_with_traffic_weights, node_zorder=2,node_size=0.03,edge_linewidth=1,edge_color=ec,node_alpha = 0.5,node_color='k', bgcolor='k',use_geom=True, axis_off=False,show=False, close=False)
# Also will take some time
df_bike_accidents_in_region['nearest_node'] = Tagger.tag_crashes(df_bike_accidents_in_region,G)
df_bike_accidents_in_region.to_csv('./CleanedData/accidents_with_nodes')
df_save2 = df_bike_accidents_in_region.copy()
df_save2.to_csv('./CleanedData/save2')
df_bike_accidents_in_region = df_save2.copy()
df_accidents_train = df_bike_accidents_in_region
add_nodes = pd.DataFrame().from_dict(add_nodes, orient='index')
edges = edges.drop(replace_edges)
edges = edges.append(add_edges)
nodes = nodes.append(add_nodes)
# reset variables to store splitting results
replace_edges = []
add_edges = {'index': [], 'geometry': []}
add_nodes = {}
stations_to_insert = recompute
recompute = []
# Add length of geometry to edge (osmnx calculates distance from node to node)
edges = edges.set_crs("EPSG:3857").to_crs("EPSG:4326")
nodes = nodes.set_crs("EPSG:3857").to_crs("EPSG:4326")
print('Adding more precise lengths')
with ProcessPoolExecutor() as executor:
edges['length'] = list(
tqdm(executor.map(length_of_line, edges['geometry']),
total=len(edges['geometry'])))
# edges['length'] = lengths
streckennetz = ox.graph_from_gdfs(nodes, edges)
streckennetz = simplify(streckennetz)
nodes, edges = ox.graph_to_gdfs(streckennetz, fill_edge_geometry=True)
print('Uploading minimal')
upload_minimal(streckennetz)
print('Uploading full')
upload_full(nodes, edges)
