def load_graphs(name):
'''
Load the graphs (bike and drive) and the CSV with the results of the algorithm into the script.
---
name: str name of the city to be loaded
layer: str layer to be loaded, it can be: drive, bike, walk, rail.
returns: Networkx MultiDiGraph
'''
crs = {
'Phoenix': {
'init': 'epsg:2763'
},
'Detroit': {
'init': 'epsg:2763'
},
'Manhattan': {
'init': 'epsg:2763'
},
'Amsterdam': {
'init': 'epsg:32633'
},
'Mexico': {
'init': 'epsg:6372'
},
'London': {
'init': 'epsg:32633'
},
'Singapore': {
'init': 'epsg:3414'
},
'Budapest': {
'init': 'epsg:32633'
},
'Copenhagen': {
'init': 'epsg:32633'
},
'Barcelona': {
'init': 'epsg:32633'
},
'Portland': {
'init': 'epsg:26949'
},
'Bogota': {
'init': 'epsg:3116'
},
'LA': {
'init': 'epsg:2763'
},
'Jakarta': {
'init': 'epsg:5331'
}
}
G_bike = ox.load_graphml('{}/{}_bike.graphml'.format(name, name))
G_drive = ox.load_graphml('{}/{}_drive.graphml'.format(name, name))
G_bike = ox.get_undirected(G_bike)
G_drive = ox.get_undirected(G_drive)
return G_bike, G_drive
def setup_graph_bbox(self, bbox):
print('+' * 15, 'setup graph from bounding box', '+' * 15)
graph_name = 'graph_' + '_'.join(
[str(v) for l in bbox.values()
for v in l]).replace('.', 'p') + '.pkl'
pickle_path = self.path_pickle + os.sep + graph_name
if os.path.isfile(pickle_path):
# Read previously downloaded graph
print('Reading pickle file', os.path.basename(pickle_path))
self.graph = pickle.load(open(pickle_path, 'rb'))
else:
print('setting up graph for bounding box',
bbox,
': downloading',
end=' ... ')
self.graph = osmnx.graph_from_bbox(
south=bbox['lat'][0],
north=bbox['lat'][1],
west=bbox['lon'][0],
east=bbox['lon'][1],
network_type='all'
) # could use bike but this might be to restrictive
print('converting to undirected graph', end=' ... ')
self.graph = osmnx.get_undirected(
self.graph
) # forget the direction, this might be to restrictive
# print('projecting to UTM', end=' ... ')
# self.graph = osmnx.project_graph(self.graph) # convert to x,y projection for faster euclidean calculations
print('saving to', pickle_path)
pickle.dump(self.graph, open(pickle_path, 'bw'))
print('-' * 42)
def street_graph_from_gdf(gdf, network_type='all'):
"""
Download streets within a convex hull around a GeoDataFrame.
Used to ensure that all streets around city blocks are downloaded, not just
those inside an arbitrary bounding box.
Parameters
----------
gdf : geodataframe
currently accepts a projected gdf
network_type : string
network_type as defined in osmnx
Returns
-------
networkx multidigraph
"""
# generate convex hull around the gdf
boundary = gdf_convex_hull(gdf)
# download the highway network within this boundary
street_graph = ox.graph_from_polygon(boundary,
network_type,
simplify=True,
retain_all=True,
truncate_by_edge=True,
clean_periphery=False)
# remove duplicates which make polygonization fail
street_graph = ox.get_undirected(street_graph)
return street_graph
def test_find_nearest():
# get graph and x/y coords to search
G = ox.graph_from_point(location_point, dist=500, network_type="drive")
Gp = ox.project_graph(G)
points = ox.utils_geo.sample_points(ox.get_undirected(Gp), 5)
X = points.x
Y = points.y
# get nearest node
nn, d = ox.get_nearest_node(Gp, location_point, method="euclidean", return_dist=True)
nn = ox.get_nearest_node(Gp, location_point, method="euclidean", return_dist=False)
# get nearest nodes
nn1, dist1 = ox.get_nearest_nodes(G, X, Y, return_dist=True)
nn2, dist2 = ox.get_nearest_nodes(Gp, X, Y, method="kdtree", return_dist=True)
nn3, dist3 = ox.get_nearest_nodes(G, X, Y, method="balltree", return_dist=True)
nn4 = ox.get_nearest_nodes(G, X, Y)
nn5 = ox.get_nearest_nodes(Gp, X, Y, method="kdtree")
nn6 = ox.get_nearest_nodes(G, X, Y, method="balltree")
# get nearest edge
u, v, k, g, d = ox.get_nearest_edge(Gp, location_point, return_geom=True, return_dist=True)
u, v, k, g = ox.get_nearest_edge(Gp, location_point, return_geom=True)
u, v, k, d = ox.get_nearest_edge(Gp, location_point, return_dist=True)
u, v, k = ox.get_nearest_edge(Gp, location_point)
# get nearest edges
ne0 = ox.distance.nearest_edges(Gp, X, Y, interpolate=50)
ne1 = ox.get_nearest_edges(Gp, X, Y)
ne2 = ox.get_nearest_edges(Gp, X, Y, method="kdtree")
ne3 = ox.get_nearest_edges(G, X, Y, method="balltree", dist=0.0001)
def setup_graph_place(self, place, which_result=1):
print('+' * 15, 'setup graph from place', '+' * 15)
graph_name = 'graph_' + place + '.pkl'
pickle_path = self.path_pickle + os.sep + graph_name
if os.path.isfile(pickle_path):
# Read previously downloaded graph
print('Reading pickle file', os.path.basename(pickle_path))
self.graph = pickle.load(open(pickle_path, 'rb'))
else:
print('setting up graph for place',
place,
': downloading',
end=' ... ')
self.graph = osmnx.graph_from_place(
query=place, which_result=which_result, network_type='all'
) # could use bike but this might be to restrictive
print('converting to undirected graph', end=' ... ')
self.graph = osmnx.get_undirected(
self.graph
) # forget the direction, this might be to restrictive
# print('projecting to UTM', end=' ... ')
# self.graph = osmnx.project_graph(self.graph) # convert to x,y projection for faster euclidean calculations
print('saving to', pickle_path)
pickle.dump(self.graph, open(pickle_path, 'bw'))
print('-' * 42)
def load_data(osm_graphml_path, osm_buffer_gpkg_path,
official_streets_gpkg_path):
"""
Load the street network edges and study boundary.
Parameters
----------
osm_graphml_path : str
path to the OSM graphml file
osm_buffer_gpkg_path : str
path to the buffered study area geopackage
official_streets_gpkg_path : str
path to the official streets shapefile
Returns
-------
gdf_osm_streets_clipped, gdf_official_streets_clipped, study_area : tuple
the osm streets (clipped to the study area), the official streets
(clipped to the study area), and the study area polygon
"""
# load the study area boundary as a shapely (multi)polygon
gdf_study_area = gpd.read_file(osm_buffer_gpkg_path,
layer="urban_study_region")
study_area = gdf_study_area["geometry"].iloc[0]
print(ox.ts(), "loaded study area boundary")
# load the official streets shapefile
gdf_official_streets = gpd.read_file(official_streets_gpkg_path)
print(ox.ts(), "loaded official streets shapefile")
# load the graph, make it undirected, then get edges GeoDataFrame
gdf_osm_streets = ox.graph_to_gdfs(ox.get_undirected(
ox.load_graphml(osm_graphml_path)),
nodes=False)
print(ox.ts(), "loaded osm edges and made undirected streets")
# Project the data to a common crs
crs = gdf_study_area.crs
if gdf_osm_streets.crs != crs:
gdf_osm_streets = gdf_osm_streets.to_crs(crs)
print(ox.ts(), "projected osm streets")
if gdf_official_streets.crs != crs:
gdf_official_streets = gdf_official_streets.to_crs(crs)
print(ox.ts(), "projected official streets")
# spatially clip the streets to the study area boundary
import warnings
warnings.filterwarnings("ignore", "GeoSeries.notna",
UserWarning) # temp warning suppression
gdf_osm_streets_clipped = gpd.clip(gdf_osm_streets, study_area)
gdf_official_streets_clipped = gpd.clip(gdf_official_streets, study_area)
print(ox.ts(), "clipped osm/official streets to study area boundary")
# double-check everything has same CRS, then return
assert gdf_osm_streets_clipped.crs == gdf_official_streets_clipped.crs == gdf_study_area.crs
return gdf_osm_streets_clipped, gdf_official_streets_clipped, study_area
def _osm_net_from_osmnx(self, boundary, osm_file=None):
"""
Submits an Overpass API query and returns a geodataframe of results
Parameters
----------
boundary : shapely geometry object
shapely geometry representing the boundary for pulling the network
osm_file : str, optional
an OSM XML file to use instead of downloading data from the network
"""
# https://osmnx.readthedocs.io/en/stable/osmnx.html#osmnx.save_load.graph_to_gdfs
node_tags = [
"access", "amenity", "bicycle", "bridge", "button_operated",
"crossing", "flashing_lights", "foot", "highway", "junction",
"leisure", "motorcar", "name", "oneway", "oneway:bicycle",
"operator", "public_transport", "railway", "segregated", "shop",
"stop", "surface", "traffic_sign", "traffic_signals", "tunnel",
"width"
]
way_tags = [
"access", "bridge", "bicycle", "button_operated", "crossing",
"cycleway", "cycleway:left", "cycleway:right", "cycleway:both",
"cycleway:buffer", "cycleway:left:buffer", "cycleway:right:buffer",
"cycleway:both:buffer", "cycleway:width", "cycleway:left:width",
"cycleway:right:width", "cycleway:both:width", "flashing_lights",
"foot", "footway", "highway", "junction", "landuse", "lanes",
"lanes:forward", "lanes:backward", "lanes:both_ways", "leisure",
"maxspeed", "motorcar", "name", "oneway", "oneway:bicycle",
"operator", "parking", "parking:lane", "parking:lane:right",
"parking:lane:left", "parking:lane:both", "parking:lane:width",
"parking:lane:right:width", "parking:lane:left:width",
"parking:lane:both:width", "public_transport", "railway",
"segregated", "service", "shop", "stop", "surface", "tracktype",
"traffic_sign", "traffic_signals:direction", "tunnel",
"turn:lanes", "turn:lanes:both_ways", "turn:lanes:backward",
"turn:lanes:forward", "width", "width:lanes",
"width:lanes:forward", "width:lanes:backward"
]
ox.config(useful_tags_node=node_tags, useful_tags_path=way_tags)
if osm_file:
G = ox.graph_from_file(osm_file, simplify=True, retain_all=False)
else:
G = ox.graph_from_polygon(boundary,
network_type='all',
simplify=True,
retain_all=False,
truncate_by_edge=False,
timeout=180,
clean_periphery=True,
custom_filter=None)
G = ox.get_undirected(G)
gdfs = ox.graph_to_gdfs(G)
return gdfs[1], gdfs[0]
def get_walkable_network(extent_poly_wgs=None):
# define filter for acquiring walkable street network
cust_filter = '["area"!~"yes"]["highway"!~"trunk_link|motor|proposed|construction|abandoned|platform|raceway"]["foot"!~"no"]["service"!~"private"]["access"!~"private"]'
# query graph
g = ox.graph_from_polygon(extent_poly_wgs, custom_filter=cust_filter)
print('loaded graph of', g.number_of_edges(), 'edges')
# convert graph to undirected graph
g_u = ox.get_undirected(g)
print('converted graph to undirected graph of', g_u.number_of_edges(),
'edges')
# project graph
g_u_proj = ox.project_graph(g_u, from_epsg(3879))
return g_u_proj
def get_unwalkable_network(extent_poly_wgs=None):
cust_filter_no_tunnels = '["area"!~"yes"]["highway"!~"trunk_link|motor|proposed|construction|abandoned|platform|raceway"]["foot"!~"no"]["service"!~"private"]["access"!~"private"]["highway"~"service"]["layer"~"-1|-2|-3|-4|-5|-6|-7"]'
# query graph
g = ox.graph_from_polygon(extent_poly_wgs,
custom_filter=cust_filter_no_tunnels,
retain_all=True)
print('loaded graph of', g.number_of_edges(), 'edges')
# convert graph to undirected graph
g_u = ox.get_undirected(g)
print('converted graph to undirected graph of', g_u.number_of_edges(),
'edges')
# project graph
g_u_proj = ox.project_graph(g_u, from_epsg(3879))
return g_u_proj
def test_stats():
# create graph, add a new node, add bearings, project it
G = ox.graph_from_point(location_point, dist=500, network_type="drive")
G.add_node(0, x=location_point[1], y=location_point[0])
_ = ox.bearing.get_bearing((0, 0), (1, 1))
G = ox.add_edge_bearings(G)
G_proj = ox.project_graph(G)
# calculate stats
cspn = ox.utils_graph.count_streets_per_node(G)
stats = ox.basic_stats(G)
stats = ox.basic_stats(G, area=1000)
stats = ox.basic_stats(G_proj,
area=1000,
clean_intersects=True,
tolerance=15,
circuity_dist="euclidean")
# calculate extended stats
stats = ox.extended_stats(G,
connectivity=True,
anc=False,
ecc=True,
bc=True,
cc=True)
# calculate entropy
Gu = ox.get_undirected(G)
entropy = ox.bearing.orientation_entropy(Gu, weight="length")
fig, ax = ox.bearing.plot_orientation(Gu, area=True, title="Title")
fig, ax = ox.bearing.plot_orientation(Gu, ax=ax, area=False, title="Title")
# test cleaning and rebuilding graph
G_clean = ox.consolidate_intersections(G_proj,
tolerance=10,
rebuild_graph=True,
dead_ends=True)
G_clean = ox.consolidate_intersections(G_proj,
tolerance=10,
rebuild_graph=True,
reconnect_edges=False)
G_clean = ox.consolidate_intersections(G_proj,
tolerance=10,
rebuild_graph=False)
# try consolidating an empty graph
G = nx.MultiDiGraph(crs="epsg:4326")
G_clean = ox.consolidate_intersections(G, rebuild_graph=True)
G_clean = ox.consolidate_intersections(G, rebuild_graph=False)
def get_orientation_entropy(G, weight=None):
Gu = ox.add_edge_bearings(ox.get_undirected(G))
if weight != None:
# weight bearings by NUMERIC attribute
city_bearings = []
for u, v, k, d in Gu.edges(keys=True, data=True):
city_bearings.extend([d['bearing']] * int(d[weight]))
b = pd.Series(city_bearings)
bearings = pd.concat([b, b.map(_reverse_bearing)
]).reset_index(drop='True')
else:
# don't weight bearings, just take one value per street segment
b = pd.Series(
[d['bearing'] for u, v, k, d in Gu.edges(keys=True, data=True)])
bearings = pd.concat([b, b.map(_reverse_bearing)
]).reset_index(drop='True')
counts = _count_and_merge(36, bearings)
probs = counts / counts.sum()
return (-np.log(probs) * probs).sum()
def get_bearings(places, weight_by_length=False):
bearings = {}
for place in sorted(places.keys()):
query = places[place]
G = ox.graph_from_place(query, network_type='drive')
Gu = ox.add_edge_bearings(ox.get_undirected(G))
if weight_by_length:
city_bearings = []
for u, v, k, d in Gu.edges(keys=True, data=True):
city_bearings.extend([d['bearing']] * int(d['length']))
b = pd.Series(city_bearings)
bearings[place] = pd.concat([b, b.map(reverse_bearing)
]).reset_index(drop='True')
else:
b = pd.Series([
d['bearing'] for u, v, k, d in Gu.edges(keys=True, data=True)
])
bearings[place] = pd.concat([b, b.map(reverse_bearing)
]).reset_index(drop='True')
return bearings
def street_graph_from_point(point, distance, network_type='all'):
"""
Use osmnx to retrieve a networkx graph from OpenStreetMap.
This function uses osmnx to obtain a graph with some set parameters. It
projects the graph and converts it to undirected. This is important - a
directed graph creates overlapping edges which shapely fails to
polygonize.
Parameters
----------
point : tuple
the (lat, lon) point to create the bounding box around
distance : int
how many meters the north, south, east, and west sides of the box should
each be from the point
network_type : string
network_type as defined in osmnx
Returns
-------
networkx multidigraph
"""
# download the highway network within this boundary
street_graph = ox.graph_from_point(point,
distance,
network_type,
simplify=True,
retain_all=True,
truncate_by_edge=True,
clean_periphery=True)
# project the network to UTM & convert to undirected graph to
street_graph = ox.project_graph(street_graph)
# remove duplicates which make polygonization fail
street_graph = ox.get_undirected(street_graph)
return street_graph
import random
from geographiclib.geodesic import Geodesic
import time
geod = Geodesic.WGS84
G = ox.graph_from_bbox(1.3763,
1.3007,
103.6492,
103.7840,
network_type='drive') #, simplify=True)
G = ox.add_edge_speeds(G)
G = ox.add_edge_travel_times(G)
projected_graph = ox.project_graph(G, to_crs="EPSG:3395")
Gc = ox.consolidate_intersections(projected_graph, dead_ends=True)
edges = ox.graph_to_gdfs(ox.get_undirected(Gc), nodes=False)
vehnumber = 10
routes = []
route = []
allAngleList = []
direction = []
all_route_roadnames = []
all_route_speeds = []
angleList = []
direzione = []
route_roadnames = []
route_speed = []
LEFT_SIG = ""
def get_map(self, start_lat, start_long, end_lat, end_long, chosen_weight):
print(
"Coordinates",
start_lat,
start_long,
end_lat,
end_long,
)
print("weight", chosen_weight)
place = 'Amherst'
place_query = {
'city': 'Amherst',
'state': 'Massachusetts',
'country': 'USA'
}
G = ox.graph_from_place(place_query, network_type='drive')
G = ox.add_node_elevations(
G, api_key='AIzaSyB9DBYn2sdIznFbmBg4DHOTl54soDBkx2E')
G = ox.add_edge_grades(G)
edge_grades = [
data['grade_abs']
for u, v, k, data in ox.get_undirected(G).edges(keys=True,
data=True)
]
avg_grade = np.mean(edge_grades)
#print('Average street grade in {} is {:.1f}%'.format(place, avg_grade*100))
if chosen_weight == 0:
choice = 'length'
elif chosen_weight == 1:
choice = 'minimum'
elif chosen_weight == 2:
choice = 'impedence'
med_grade = np.median(edge_grades)
#print('Median street grade in {} is {:.1f}%'.format(place, med_grade*100))
# project the street network to UTM
G_proj = ox.project_graph(G)
# get one color for each node, by elevation, then plot the network
#nc = ox.get_node_colors_by_attr(G_proj, 'elevation', cmap='plasma', num_bins=20)
#fig, ax = ox.plot_graph(G_proj, fig_height=6, node_color=nc, node_size=12, node_zorder=2, edge_color='#dddddd')
# get a color for each edge, by grade, then plot the network
#ec = ox.get_edge_colors_by_attr(G_proj, 'grade_abs', cmap='plasma', num_bins=10)
#fig, ax = ox.plot_graph(G_proj, fig_height=6, edge_color=ec, edge_linewidth=0.8, node_size=0)
# select an origin and destination node and a bounding box around them
origin = ox.get_nearest_node(G, (start_lat, start_long))
destination = ox.get_nearest_node(G, (end_lat, end_long))
bbox = ox.bbox_from_point(
((start_lat + end_lat) / 2, (start_long + end_long) / 2),
distance=5000,
project_utm=True)
for u, v, k, data in G_proj.edges(keys=True, data=True):
data['impedance'] = self.impedance(data['length'],
data['grade_abs'])
data['rise'] = data['length'] * data['grade']
#weight_choice = {'easy' : 'length', 'median' : 'minimum', 'hard' : 'impedance'}
routef = nx.shortest_path(G_proj,
source=origin,
target=destination,
weight=choice)
route_map = ox.plot_route_folium(G, routef)
p1 = [start_lat, start_long]
p2 = [end_lat, end_long]
folium.Marker(location=p1,
icon=folium.Icon(color='green')).add_to(route_map)
folium.Marker(location=p2,
icon=folium.Icon(color='red')).add_to(route_map)
print("------------------4321")
result = self.print_route_stats(routef, G_proj)
filepath = 'routeff.html'
route_map.save(filepath)
IFrame(filepath, width=600, height=500)
return result
# Graph for Walkability analysis should not be directed
# (ie. it is assumed pedestrians are not influenced by one way streets)
# note that when you save the undirected G_proj feature, if you re-open it, it is directed again
#
# >>> G_proj = ox.load_graphml(proj_graphml_filepath)
# >>> nx.is_directed(G_proj)
# True
# >>> G_proj = ox.get_undirected(G_proj)
# >>> nx.is_directed(G_proj)
# False
# >>> ox.save_graphml(G_proj, proj_graphml_filepath)
# >>> G_proj = ox.load_graphml(proj_graphml_filepath)
# >>> nx.is_directed(G_proj)
# True
# so no point undirecting it before saving - you have to undirect again regardless
G_proj = ox.get_undirected(G_proj)
# read search distance from json file, the default should be 1600m
# the search distance is used to defined the radius of a sample point as a local neighborhood
distance = sc.parameters["search_distance"]
# get the nodes GeoDataFrame row length for use in later iteration
rows = gdf_nodes_simple.shape[0]
# if provide 'true' in command line, then using multiprocessing, otherwise, using single thread
# Notice: Meloubrne has the largest number of sample points, which needs 13 GB memory for docker using 3 cpus.
if len(sys.argv) > 2:
if sys.argv[2].lower() == "true":
# method1: new way to use multiprocessing
# get a list of nodes id for later iteration purpose
place = 'Amherst'
place_query = {'city':'Amherst', 'state':'Massachusetts', 'country':'USA'}
G = ox.graph_from_place(place_query, network_type='drive')
# In[21]:
G = ox.add_node_elevations(G, api_key=google_elevation_api_key)
G = ox.add_edge_grades(G)
# In[22]:
edge_grades = [data['grade_abs'] for u, v, k, data in ox.get_undirected(G).edges(keys=True, data=True)]
# In[23]:
avg_grade = np.mean(edge_grades)
print('Average street grade in {} is {:.1f}%'.format(place, avg_grade*100))
med_grade = np.median(edge_grades)
print('Median street grade in {} is {:.1f}%'.format(place, med_grade*100))
# In[24]:
import utils.networks as nw
from multiprocessing import current_process, Pool
import networkx as nx
import time
# READ EXTENT
hel_poly = files.get_hel_poly()
hel_poly_buff = hel_poly.buffer(1500)
extent = geom_utils.project_to_wgs(hel_poly_buff)
# GET GRAPH
graph = nw.get_walk_network(extent)
ox.save_graphml(graph, filename='hel.graphml', folder='graphs', gephi=False)
# UNDIRECTED
graph_u = ox.get_undirected(graph)
ox.save_graphml(graph_u,
filename='hel_u.graphml',
folder='graphs',
gephi=False)
# GET EDGE DICTS
edge_dicts = nw.get_all_edge_dicts(graph_u)
print('Edges in the graph:', len(edge_dicts))
edge_dicts[:2]
# MISSING GEOM ADDED
def get_edge_geoms(edge_dict):
return nw.get_missing_edge_geometries(edge_dict, graph_u)
def calculate_graph_indicators(graphml_folder, country_folder, filename):
# get filepath and country/city identifiers
filepath = os.path.join(graphml_folder, country_folder, filename)
country, country_iso = country_folder.split('-')
core_city, uc_id = filename.replace('.graphml', '').split('-')
uc_id = int(uc_id)
start_time = time.time()
print(ox.ts(), 'processing', filepath)
G = ox.load_graphml(filepath=filepath)
# clustering and pagerank: needs directed representation
cc_avg_dir, cc_avg_undir, cc_wt_avg_dir, cc_wt_avg_undir, pagerank_max = get_clustering(
G)
# get an undirected representation of this network for everything else
Gu = ox.get_undirected(G)
G.clear()
G = None
# street lengths
lengths = pd.Series(nx.get_edge_attributes(Gu, 'length'))
length_total = lengths.sum()
length_median = lengths.median()
length_mean = lengths.mean()
# nodes, edges, node degree, self loops
n = len(Gu.nodes)
m = len(Gu.edges)
k_avg = 2 * m / n
self_loop_proportion = sum(u == v for u, v, k in Gu.edges) / m
# proportion of 4-way intersections, 3-ways, and dead-ends
streets_per_node = nx.get_node_attributes(Gu, 'street_count')
prop_4way = list(streets_per_node.values()).count(4) / n
prop_3way = list(streets_per_node.values()).count(3) / n
prop_deadend = list(streets_per_node.values()).count(1) / n
# average circuity and straightness
circuity = calculate_circuity(Gu, length_total)
straightness = 1 / circuity
# elevation and grade
grade_mean, grade_median, elev_mean, elev_median, elev_std, elev_range, elev_iqr = elevation_grades(
Gu)
# bearing/orientation entropy/order
orientation_entropy = calculate_orientation_entropy(Gu)
orientation_order = calculate_orientation_order(orientation_entropy)
# total and clean intersection counts
intersect_count, intersect_count_clean, intersect_count_clean_topo = intersection_counts(
ox.project_graph(Gu), streets_per_node)
# assemble the results
rslt = {
'country': country,
'country_iso': country_iso,
'core_city': core_city,
'uc_id': uc_id,
'cc_avg_dir': cc_avg_dir,
'cc_avg_undir': cc_avg_undir,
'cc_wt_avg_dir': cc_wt_avg_dir,
'cc_wt_avg_undir': cc_wt_avg_undir,
'circuity': circuity,
'elev_iqr': elev_iqr,
'elev_mean': elev_mean,
'elev_median': elev_median,
'elev_range': elev_range,
'elev_std': elev_std,
'grade_mean': grade_mean,
'grade_median': grade_median,
'intersect_count': intersect_count,
'intersect_count_clean': intersect_count_clean,
'intersect_count_clean_topo': intersect_count_clean_topo,
'k_avg': k_avg,
'length_mean': length_mean,
'length_median': length_median,
'length_total': length_total,
'street_segment_count': m,
'node_count': n,
'orientation_entropy': orientation_entropy,
'orientation_order': orientation_order,
'pagerank_max': pagerank_max,
'prop_4way': prop_4way,
'prop_3way': prop_3way,
'prop_deadend': prop_deadend,
'self_loop_proportion': self_loop_proportion,
'straightness': straightness
}
elapsed = time.time() - start_time
ox.log(f'finished {filepath} in {elapsed:.0f} seconds')
return rslt
def load_PEMS(num_train=250, dtype=np.float64):
#unzip daata
with zipfile.ZipFile('data/PEMS.zip', 'r') as zip_ref:
zip_ref.extractall('data')
# Data reading
with open('data/PEMS/adj_mx_bay.pkl', 'rb') as f:
sensor_ids, sensor_id_to_ind, _ = pickle.load(f, encoding='latin1')
all_signals = pd.read_hdf('data/PEMS/pems-bay.h5')
coords = pd.read_csv('data/PEMS/graph_sensor_locations_bay.csv',
header=None)
# Loading real world graph of roads
north, south, east, west = 37.450, 37.210, -121.80, -122.10
if not os.path.isfile('data/PEMS/bay_graph.pkl'):
cf = '["highway"~"motorway|motorway_link"]' # Road filter, we don't use small ones.
G = osmnx.graph_from_bbox(north=north,
south=south,
east=east,
west=west,
simplify=True,
custom_filter=cf)
with open(
'data/PEMS/bay_graph.pkl',
'wb') as f: # frequent loading of maps leads to a temporal ban
pickle.dump(G, f)
else:
with open('data/PEMS/bay_graph.pkl', 'rb') as f:
G = pickle.load(f)
G = osmnx.get_undirected(G) # Matern GP supports only undirected graphs.
# Graph cleaning up
for _ in range(2):
out_degree = G.degree
to_remove = [node for node in G.nodes if out_degree[node] == 1]
G.remove_nodes_from(to_remove)
G = nx.convert_node_labels_to_integers(G)
G.remove_nodes_from([372, 286])
G = nx.convert_node_labels_to_integers(G)
num_points = len(sensor_ids)
np_coords = np.zeros((num_points, 2)) # Vector of sensors coordinates.
for i in range(num_points):
sensor_id, x, y = coords.iloc[i]
ind = sensor_id_to_ind[str(int(sensor_id))]
np_coords[ind][0], np_coords[ind][1] = x, y
coords = np_coords
sensor_ind_to_node = {}
# Inserting sensors into a graph. During insertion, the edge containing the sensor is cut
for point_id in range(num_points):
sensor_ind_to_node[point_id] = len(G) # adding new vertex at the end
sensor_point = Point(coords[point_id, 1], coords[point_id, 0])
u, v, key, geom = osmnx.get_nearest_edge(
G, (sensor_point.y, sensor_point.x), return_geom=True)
edge = G.edges[(u, v, key)]
G.remove_edge(u, v, key)
edge_1_geom, edge_2_geom = cut(geom, geom.project(sensor_point))
l_ratio = geom.project(sensor_point, normalized=True)
l_1, l_2 = l_ratio * edge['length'], (1 - l_ratio) * edge['length']
new_vertex = nearest_points(geom, sensor_point)[0]
G.add_node(len(G), x=new_vertex.x, y=new_vertex.y)
G.add_edge(u, len(G) - 1, length=l_1, geometry=edge_1_geom)
G.add_edge(len(G) - 1, v, length=l_2, geometry=edge_2_geom)
# Weights are inversely proportional to the length of the road
lengths = nx.get_edge_attributes(G, 'length')
lengths_list = [length for length in lengths.values()]
mean_length = np.mean(lengths_list)
weights = {}
for edge, length in lengths.items():
weights[edge] = mean_length / length
nx.set_edge_attributes(G, values=weights, name='weight')
# Sorry for that,
# sensor_ind - sensor id in California database, sensor_id - local numeration (from 1 to num of sensors)
sensor_id_to_node = {}
for sensor_ind, node in sensor_ind_to_node.items():
sensor_id = sensor_ids[sensor_ind]
sensor_id_to_node[sensor_id] = node
# Selecting signals at some moment
signals = all_signals[(all_signals.index.weekday == 0)
& (all_signals.index.hour == 17) &
(all_signals.index.minute == 30)]
# Dataset creation
x, y = [], []
for i in range(len(signals)):
for sensor_id in sensor_ids:
if sensor_id_to_node.get(sensor_id) is not None:
node = sensor_id_to_node[sensor_id]
signal = signals.iloc[i][int(sensor_id)]
x.append([i, node])
y.append([signal])
x, y = np.asarray(x, dtype=dtype), np.asarray(y, dtype=dtype)
x, y = x[:num_points, 1:], y[:num_points]
# Splitting data into train and test
random_perm = np.random.permutation(np.arange(x.shape[0]))
train_vertex, test_vertex = random_perm[:num_train], random_perm[
num_train:]
x_train, x_test = x[train_vertex], x[test_vertex]
y_train, y_test = y[train_vertex], y[test_vertex]
return G, (x_train, y_train), (x_test, y_test), (x, y)
'unclassified', 'road'
]
minor_streets = [(u, v, k) for u, v, k, d in G.edges(keys=True, data=True)
if d['highway'] not in types]
# remove minor streets and retain only the largest connected component subgraph
G_ter = G
G_ter.remove_edges_from(minor_streets)
G_ter = ox.remove_isolated_nodes(G_ter)
G_ter_connected = ox.get_largest_component(G_ter, strongly=True)
# then simplify the graph now that we have only the edge types we want
G_ter_simp = ox.simplify_graph(G_ter_connected, strict=True)
# create a unique ID for each edge because osmid can
# hold multiple values due to topology simplification
i = 0
for u, v, k, d in G_ter_simp.edges(data=True, keys=True):
d['uniqueid'] = i
i += 1
# convert to two-way
H = ox.get_undirected(G_ter_simp)
# save graph as OSM
ox.save_graph_osm(
H,
oneway=False,
filename='bay_area_simplified_tertiary_strongly_2_way_network.osm')
return x + 180 if x < 180 else x - 180
from datetime import datetime
start = datetime.now()
bearings = {}
for place in sorted(places.keys()):
# get the graph
query = places[place]
G = ox.graph_from_place(query, network_type='drive')
# calculate edge bearings
Gu = ox.add_edge_bearings(ox.get_undirected(G))
if weight_by_length:
# weight bearings by length (meters)
city_bearings = []
for u, v, k, d in Gu.edges(keys=True, data=True):
city_bearings.extend([d['bearing']] * int(d['length']))
b = pd.Series(city_bearings)
bearings[place] = pd.concat([b, b.map(reverse_bearing)
]).reset_index(drop='True')
else:
# don't weight bearings, just take one value per street segment
b = pd.Series(
[d['bearing'] for u, v, k, d in Gu.edges(keys=True, data=True)])
bearings[place] = pd.concat([b, b.map(reverse_bearing)
import osmnx as ox
import pathlib
map_location = "Sydney"
path = pathlib.Path(__file__).parent.absolute()
if map_location == "Cambridge":
filepath = path / "Cambridge_Graph.xml"
graph = ox.get_undirected(ox.project_graph(ox.graph_from_point((52.206000250695205, 0.1218685443020611),dist=2000, network_type="drive_service"), to_crs="EPSG:27700"))
ox.save_graphml(graph, filepath=filepath)
elif map_location == "Sydney":
filepath = path / "Sydney_Graph.xml"
graph = ox.get_undirected(ox.project_graph(ox.graph_from_address('Sydney, Australia',dist=1500, network_type="drive_service"), to_crs="EPSG:27700"))
ox.save_graphml(graph, filepath=filepath)
