def download_osm_features(place, osm_type, tag, values=None, by_poly=True, timeout=180):
""" Download OSM features within given place
:param place: single place name query (e.g: "London", "Bonn", etc.)
:param osm_type: OSM geometry type str ('node', 'way', 'relation')
:param tag: OSM tag to query
:param values: str/list of possible values for the provided OSM tag
:param by_poly: if True, retrieve features within polygon's list of coordinates, otherwise use bounds
:param timeout:
:return:
"""
gdf_geometry = geocode_to_gdf(place)
try:
geometry = gdf_geometry.geometry[0]
except AttributeError: # Empty GeoDataFrame
return None
responses = []
if by_poly:
polygon_coord_strs = get_polygons_coordinates(geometry)
for poly_coord_str in polygon_coord_strs:
query = ql_query(osm_type, tag, values, polygon_coord=poly_coord_str, timeout=timeout)
responses.append(overpass_request(data={'data': query}))
else:
query = ql_query(osm_type, tag, values, bounds=geometry.bounds, timeout=timeout)
responses.append(overpass_request(data={'data': query}))
return responses
len(list(G.edges())),
time.time() - start_time))
# add length (great circle distance between nodes) attribute to each edge to
# use as weight
if len(G.edges) > 0:
G = ox.add_edge_lengths(G)
return G
punggol = (1.403948, 103.909048)
distance = 2000
G_walk = ox.graph_from_point(punggol, distance=distance, truncate_by_edge=True, network_type='drive', simplify=False)
mrt_query_str = '[out:json][timeout:180];(relation["network"="Singapore Rail"]["route"="monorail"](1.4011,103.8977,1.4154,103.9231);>;);out;'
mrt_response_json = overpass_request(data={'data': mrt_query_str}, timeout=180)
G_lrt = create_graph(mrt_response_json)
# storing all nodes into a list
walkNodeList = list(G_walk.nodes.values())
mrtNodeList = list(G_lrt.nodes.values())
mrtEdgeList = list(G_lrt.edges.items())
pe = []
pw = []
for k in mrtNodeList: # check for nodes which are stations
try:
if "PE" in k.get('ref'):
pe.append(k.get('osmid'))
if "PW" in k.get('ref'):
def overpass_request(data,
pause_duration=None,
timeout=180,
error_pause_duration=None):
"""
Send a request to the Overpass API via HTTP POST and return the JSON
response.
Parameters
----------
data : dict or OrderedDict
key-value pairs of parameters to post to the API
pause_duration : int
how long to pause in seconds before requests, if None, will query API
status endpoint to find when next slot is available
timeout : int
the timeout interval for the requests library
error_pause_duration : int
how long to pause in seconds before re-trying requests if error
Returns
-------
dict
"""
# define the Overpass API URL, then construct a GET-style URL as a string to
# hash to look up/save to cache
url = settings.overpass_endpoint.rstrip('/') + '/interpreter'
prepared_url = requests.Request('GET', url,
params=data).prepare().url
cached_response_json = get_from_cache(prepared_url)
if cached_response_json is not None:
# found this request in the cache, just return it instead of making a
# new HTTP call
return cached_response_json
else:
# if this URL is not already in the cache, pause, then request it
if pause_duration is None:
this_pause_duration = get_pause_duration()
log('Pausing {:,.2f} seconds before making API POST request'.
format(this_pause_duration))
time.sleep(this_pause_duration)
start_time = time.time()
log('Posting to {} with timeout={}, "{}"'.format(
url, timeout, data))
response = requests.post(url,
data=data,
timeout=timeout,
headers=get_http_headers())
# get the response size and the domain, log result
size_kb = len(response.content) / 1000.
domain = re.findall(r'(?s)//(.*?)/', url)[0]
log('Downloaded {:,.1f}KB from {} in {:,.2f} seconds'.format(
size_kb, domain,
time.time() - start_time))
try:
response_json = response.json()
if 'remark' in response_json:
log('Server remark: "{}"'.format(
response_json['remark'], level=lg.WARNING))
save_to_cache(prepared_url, response_json)
except Exception:
# 429 is 'too many requests' and 504 is 'gateway timeout' from server
# overload - handle these errors by recursively calling
# overpass_request until we get a valid response
if response.status_code in [429, 504]:
# pause for error_pause_duration seconds before re-trying request
if error_pause_duration is None:
error_pause_duration = get_pause_duration()
log('Server at {} returned status code {} and no JSON data. Re-trying request in {:.2f} seconds.'
.format(domain, response.status_code,
error_pause_duration),
level=lg.WARNING)
time.sleep(error_pause_duration)
response_json = overpass_request(
data=data,
pause_duration=pause_duration,
timeout=timeout)
# else, this was an unhandled status_code, throw an exception
else:
log('Server at {} returned status code {} and no JSON data'
.format(domain, response.status_code),
level=lg.ERROR)
raise Exception(
'Server returned no JSON data.\n{} {}\n{}'.format(
response, response.reason, response.text))
return response_json
def bus_layer(start, end, results, case):
"""
It generates a bus route with the bus numbers via greedy algorithm
Parameters
----------
start : node id
end : node id
results : dict (From lta datamall)
case : int
Returns
-------
final_route_list : list
"""
def overpass_request(data,
pause_duration=None,
timeout=180,
error_pause_duration=None):
"""
Send a request to the Overpass API via HTTP POST and return the JSON
response.
Parameters
----------
data : dict or OrderedDict
key-value pairs of parameters to post to the API
pause_duration : int
how long to pause in seconds before requests, if None, will query API
status endpoint to find when next slot is available
timeout : int
the timeout interval for the requests library
error_pause_duration : int
how long to pause in seconds before re-trying requests if error
Returns
-------
dict
"""
# define the Overpass API URL, then construct a GET-style URL as a string to
# hash to look up/save to cache
url = settings.overpass_endpoint.rstrip('/') + '/interpreter'
prepared_url = requests.Request('GET', url,
params=data).prepare().url
cached_response_json = get_from_cache(prepared_url)
if cached_response_json is not None:
# found this request in the cache, just return it instead of making a
# new HTTP call
return cached_response_json
else:
# if this URL is not already in the cache, pause, then request it
if pause_duration is None:
this_pause_duration = get_pause_duration()
log('Pausing {:,.2f} seconds before making API POST request'.
format(this_pause_duration))
time.sleep(this_pause_duration)
start_time = time.time()
log('Posting to {} with timeout={}, "{}"'.format(
url, timeout, data))
response = requests.post(url,
data=data,
timeout=timeout,
headers=get_http_headers())
# get the response size and the domain, log result
size_kb = len(response.content) / 1000.
domain = re.findall(r'(?s)//(.*?)/', url)[0]
log('Downloaded {:,.1f}KB from {} in {:,.2f} seconds'.format(
size_kb, domain,
time.time() - start_time))
try:
response_json = response.json()
if 'remark' in response_json:
log('Server remark: "{}"'.format(
response_json['remark'], level=lg.WARNING))
save_to_cache(prepared_url, response_json)
except Exception:
# 429 is 'too many requests' and 504 is 'gateway timeout' from server
# overload - handle these errors by recursively calling
# overpass_request until we get a valid response
if response.status_code in [429, 504]:
# pause for error_pause_duration seconds before re-trying request
if error_pause_duration is None:
error_pause_duration = get_pause_duration()
log('Server at {} returned status code {} and no JSON data. Re-trying request in {:.2f} seconds.'
.format(domain, response.status_code,
error_pause_duration),
level=lg.WARNING)
time.sleep(error_pause_duration)
response_json = overpass_request(
data=data,
pause_duration=pause_duration,
timeout=timeout)
# else, this was an unhandled status_code, throw an exception
else:
log('Server at {} returned status code {} and no JSON data'
.format(domain, response.status_code),
level=lg.ERROR)
raise Exception(
'Server returned no JSON data.\n{} {}\n{}'.format(
response, response.reason, response.text))
return response_json
def get_node(element):
"""
Convert an OSM node element into the format for a networkx node.
Parameters
----------
element : dict
an OSM node element
Returns
-------
dict
"""
useful_tags_node = ['ref', 'highway', 'route_ref', 'asset_ref']
node = {}
node['y'] = element['lat']
node['x'] = element['lon']
node['osmid'] = element['id']
if 'tags' in element:
for useful_tag in useful_tags_node:
if useful_tag in element['tags']:
node[useful_tag] = element['tags'][useful_tag]
return node
def get_path(element, element_r):
"""
Convert an OSM way element into the format for a networkx graph path.
Parameters
----------
element : dict
an OSM way element
element_r : dict
an OSM way element
Returns
-------
dict
"""
useful_tags_path_e = [
'bridge', 'tunnel', 'oneway', 'lanes', 'name', 'highway',
'maxspeed', 'service', 'access', 'area', 'landuse', 'width',
'est_width', 'junction'
]
useful_tags_path_r = [
'bridge', 'tunnel', 'oneway', 'lanes', 'ref', 'direction',
'from', 'to', 'name', 'highway', 'maxspeed', 'service',
'access', 'area', 'landuse', 'width', 'est_width', 'junction'
]
path = {}
path['osmid'] = element['id']
# remove any consecutive duplicate elements in the list of nodes
grouped_list = groupby(element['nodes'])
path['nodes'] = [group[0] for group in grouped_list]
if 'tags' in element:
# for relation in element_r['elements']:
# if relation['type'] == 'relation':
# for members in relation['members']:
# if members['ref'] == element['id']:
for useful_tag in useful_tags_path_e:
if useful_tag in element['tags']:
path[useful_tag] = element['tags'][useful_tag]
# for useful_tag in useful_tags_path_r:
# if useful_tag in relation['tags']:
# try:
# path[useful_tag] = path[useful_tag] + ";" + relation['tags'][useful_tag]
# except KeyError:
# path[useful_tag] = relation['tags'][useful_tag]
# pass
return path
def parse_osm_nodes_paths(osm_data):
"""
Construct dicts of nodes and paths with key=osmid and value=dict of
attributes.
Parameters
----------
osm_data : dict
JSON response from from the Overpass API
Returns
-------
nodes, paths : tuple
"""
nodes = {}
paths = {}
relation = {}
# for element in osm_data['elements']:
# if element['type'] == 'relation':
for element in osm_data['elements']:
if element['type'] == 'node':
key = element['id']
nodes[key] = get_node(element)
elif element['type'] == 'way': #osm calls network paths 'ways'
key = element['id']
# pp.pprint(element)
paths[key] = get_path(element, osm_data)
return nodes, paths
def create_graph(response_jsons,
name='unnamed',
retain_all=True,
bidirectional=False):
"""
Create a networkx graph from Overpass API HTTP response objects.
Parameters
----------
response_jsons : list
list of dicts of JSON responses from from the Overpass API
name : string
the name of the graph
retain_all : bool
if True, return the entire graph even if it is not connected
bidirectional : bool
if True, create bidirectional edges for one-way streets
Returns
-------
networkx multidigraph
"""
log('Creating networkx graph from downloaded OSM data...')
start_time = time.time()
# make sure we got data back from the server requests
elements = []
# for response_json in response_jsons:
elements.extend(response_json['elements'])
if len(elements) < 1:
raise EmptyOverpassResponse(
'There are no data elements in the response JSON objects')
# create the graph as a MultiDiGraph and set the original CRS to default_crs
G = nx.MultiDiGraph(name=name, crs=settings.default_crs)
# extract nodes and paths from the downloaded osm data
nodes = {}
paths = {}
# for osm_data in response_jsons:
nodes_temp, paths_temp = parse_osm_nodes_paths(response_jsons)
for key, value in nodes_temp.items():
nodes[key] = value
for key, value in paths_temp.items():
paths[key] = value
# add each osm node to the graph
for node, data in nodes.items():
G.add_node(node, **data)
# add each osm way (aka, path) to the graph
G = ox.add_paths(G, paths, bidirectional=bidirectional)
# retain only the largest connected component, if caller did not
# set retain_all=True
if not retain_all:
G = get_largest_component(G)
log('Created graph with {:,} nodes and {:,} edges in {:,.2f} seconds'
.format(len(list(G.nodes())), len(list(G.edges())),
time.time() - start_time))
# add length (great circle distance between nodes) attribute to each edge to
# use as weight
if len(G.edges) > 0:
G = ox.add_edge_lengths(G)
return G
def calculate_H(s_lat, s_lon, e_lat, e_lon):
"""
Calculate a distance with x,y coordinates with
Parameters
----------
s_lat : float (starting lat)
s_lon : float (starting lon)
e_lat : float (ending lat)
e_lon : float (ending lon)
Returns
-------
distance
"""
R = 6371.0
snlat = radians(s_lat)
snlon = radians(s_lon)
elat = radians(e_lat)
elon = radians(e_lon)
actual_dist = 6371.01 * acos(
sin(snlat) * sin(elat) +
cos(snlat) * cos(elat) * cos(snlon - elon))
actual_dist = actual_dist * 1000
return actual_dist
def bus_details_SD(adjacent_list):
"""
store all details from LTA data mall into dictionary
Parameters
----------
adjacent_list : dict
Returns
-------
adjacent_list : dict
"""
temp = 0
for x in results:
if temp != x.get('ServiceNo'):
temp = x.get('ServiceNo')
count = 0
adja_bus_stop = my_dictionary()
adjacent_list.add(temp, adja_bus_stop)
adja_bus_stop.add(
count, [x.get('BusStopCode'),
x.get('Distance')])
count += 1
else:
adja_bus_stop.add(
count, [x.get('BusStopCode'),
x.get('Distance')])
count += 1
return adjacent_list
def get_nearestedge_node(osm_id, a, G):
"""
Find the nearest node available in Open street map
Parameters
----------
osm_id : node ID
a : plotting graph
g : bus graph
Returns
-------
temp_nearest_edge[1]/temp_nearest_edge[2] : nearest node to a way ID
"""
temp_y = G.nodes.get(osm_id).get('y')
temp_x = G.nodes.get(osm_id).get('x')
temp_nearest_edge = ox.get_nearest_edge(a, (temp_y, temp_x))
temp_1 = temp_nearest_edge[0].coords[0]
temp_2 = temp_nearest_edge[0].coords[1]
temp1_x = temp_1[0]
temp1_y = temp_1[1]
temp_1_distance = calculate_H(temp1_y, temp1_x, temp_y, temp_x)
temp2_x = temp_2[0]
temp2_y = temp_2[1]
temp_2_distance = calculate_H(temp2_y, temp2_x, temp_y, temp_x)
if temp_1_distance < temp_2_distance:
return temp_nearest_edge[1]
else:
return temp_nearest_edge[2]
def delete_duplicate(x):
"""
Delete duplicate within a list
Parameters
----------
x : list
Returns
-------
list
"""
return list(dict.fromkeys(x))
def request_busG():
"""
Find all nodes that is a bus stop
Returns
-------
busG : dict
"""
busG = {}
for x in G.nodes.items():
if x[1].get('highway') == 'bus_stop':
xy = []
xy.append(x[1].get('osmid'))
xy.append(x[1].get('x'))
xy.append(x[1].get('y'))
busG[x[1].get('osmid')] = xy
return busG
# ---MAIN---#
query_str = '[out:json][timeout:180];node["type"="route"](1.385700,103.887300,1.422000,103.925900);way["type"="route"](1.385700,103.887300,1.422000,103.925900);(relation["type"="route"](1.385700,103.887300,1.422000,103.925900);>;);out;'
response_json = overpass_request(data={'data': query_str}, timeout=180)
pp = pprint.PrettyPrinter(indent=4)
# start = 1847853709
# end = 410472575
# end = 3737148763
# bus transit
# start = 2110621974
# end = 2085845884
adjacent_list = my_dictionary()
G = ox.load_graphml('Bus_Overpass.graphml')
if case == 1:
return request_busG()
n, e = ox.graph_to_gdfs(G)
# e.to_csv("Edge_test_busstop.csv")
if len(results) == 0:
results = bus_details_all(
results
) # Details from LTA Datamall, extracting all details such as service no, bus stop number
adjacent_list = bus_details_SD(
adjacent_list
) # From results, it extracts bus stop number and distance
start_busstop = (G.nodes.get(start)).get('asset_ref')
end_busstop = (G.nodes.get(end)).get('asset_ref')
#Start finding common bus service within the start bus stop and end bus stop
try:
if ";" in (G.nodes.get(start).get('route_ref')):
start_rr = (G.nodes.get(start).get('route_ref')).split(";")
else:
start_rr = []
start_rr.append((G.nodes.get(start).get('route_ref')))
print("TEST - G.nodes.get(end): ", G.nodes.get(end))
if ";" in (G.nodes.get(end).get('route_ref')):
end_rr = (G.nodes.get(end).get('route_ref')).split(";")
else:
end_rr = []
end_rr.append((G.nodes.get(end).get('route_ref')))
common = list(set(start_rr) & set(end_rr))
except:
return -1
"""
This method strictly emphasis on greedy algorithm. Thus it will prioritze the numbers of transit rather than distance
Check if any common bus service within start and end bus stop.
If found, route_list will capture the entire route of the common bus service
No transit will occuer as it is a straight path, start busstop -> end busstop
If not found, the program will proceed to find a common bus stop within the start and end bus services.
Thus a transit will occur, start busstop -> mid busstop -> end busstop
"""
route_list = {}
mid_route_list = {}
# print("TEST - Start: ", start_busstop)
# print("TEST - End: ", end_busstop)
# print("TEST - start_rr: ", start_rr)
# print("TEST - end_rr: ", end_rr)
# print("TEST - Common: ", common)
common_mid = []
if len(common) == 0: #No common bus service found
while (len(common_mid) == 0): #Start finding a common mid busstop
rona_one = []
rona_two = []
for start_to_mid in start_rr: #Capture all common mid busstop
print("TEST - start_to_mid: ", start_to_mid)
for bus_sequence in adjacent_list.get(start_to_mid):
rona_one.append(
str(
adjacent_list.get(start_to_mid).get(
bus_sequence)[0]))
for mid_to_end in end_rr:
print("TEST - mid_to_end: ", mid_to_end)
for bus_sequence in adjacent_list.get(mid_to_end):
rona_two.append(
str(
adjacent_list.get(mid_to_end).get(
bus_sequence)[0]))
found_br = []
print("TEST rona 1:", rona_one)
print("TEST rona 2:", rona_two)
found_br.append(start_to_mid + ";" + mid_to_end)
found_br.extend(list(set(rona_one) & set(rona_two)))
common_mid.append(found_br)
print("TEST - common_mid: ", common_mid)
bus_service = start_to_mid
temp_bus = []
mid_busstop = 0
approved = 0
for bus_sequence in adjacent_list.get(
bus_service
): #Finding bus service for start busstop -> mid busstop
for x in range(0, len(common_mid)):
for i in common_mid[x]:
if str(
adjacent_list.get(bus_service).get(
bus_sequence)[0]) == str(
start_busstop):
temp_bus.append(
adjacent_list.get(bus_service).get(
bus_sequence)[0])
approved = 1
if str(
adjacent_list.get(bus_service).get(
bus_sequence)[0]) == str(
i) and approved == 1:
mid_busstop = str(i)
temp_bus.append(
adjacent_list.get(bus_service).get(
bus_sequence)[0])
approved = 0
break
if approved == 1:
temp_bus.append(
adjacent_list.get(bus_service).get(
bus_sequence)[0])
if mid_busstop != 0:
break
if str(start_busstop) not in temp_bus or str(
mid_busstop
) not in temp_bus: #If not found, continue to next loop
continue
temp_bus = delete_duplicate(temp_bus)
mid_route_list[bus_service] = temp_bus
for x in G.nodes: #After finding bus service to mid busstop, start finding path mid busstop to end busstop
if G.nodes.get(x).get('asset_ref') == mid_busstop:
if ";" in (G.nodes.get(x).get('route_ref')):
start_rr = (G.nodes.get(x).get('route_ref')).split(";")
else:
start_rr = []
start_rr.append((G.nodes.get(start).get('route_ref')))
common = list(set(start_rr) & set(end_rr))
start_busstop = mid_busstop
if start == 1847853709: #If bus service started from punggol interchange
for bus_service in common:
temp_bus = []
approved = 0
for bus_sequence in adjacent_list.get(
bus_service): #Capture bus route
if str(
adjacent_list.get(bus_service).get(bus_sequence)
[0]) == str(start_busstop) and adjacent_list.get(
bus_service).get(bus_sequence)[1] == 0:
temp_bus.append(
adjacent_list.get(bus_service).get(bus_sequence)
[0])
approved = 1
if str(
adjacent_list.get(bus_service).get(bus_sequence)
[0]) == str(end_busstop) and approved == 1:
temp_bus.append(
adjacent_list.get(bus_service).get(bus_sequence)
[0])
approved = 0
break
if approved == 1:
temp_bus.append(
adjacent_list.get(bus_service).get(bus_sequence)
[0])
if str(start_busstop) not in temp_bus or str(
end_busstop) not in temp_bus:
continue
route_list[bus_service] = temp_bus
else:
for bus_service in common: #If bus service does not start from punggol interchange
temp_bus = []
approved = 0
for bus_sequence in adjacent_list.get(
bus_service): #Capture bus route
if str(
adjacent_list.get(bus_service).get(bus_sequence)
[0]) == str(start_busstop):
temp_bus.append(
adjacent_list.get(bus_service).get(bus_sequence)
[0])
approved = 1
if str(
adjacent_list.get(bus_service).get(bus_sequence)
[0]) == str(end_busstop) and approved == 1:
temp_bus.append(
adjacent_list.get(bus_service).get(bus_sequence)
[0])
approved = 0
break
if approved == 1:
temp_bus.append(
adjacent_list.get(bus_service).get(bus_sequence)
[0])
if str(start_busstop) not in temp_bus or str(
end_busstop) not in temp_bus:
continue
route_list[bus_service] = temp_bus
"""
After capturing all the bus serivce. A comparison is made in favor for the number of bus stops
It will choose the least amount of bus stops and store in post_compare
"""
compare = [0, 100]
if len(route_list.keys()) > 1:
for i in route_list:
if len(route_list.get(i)) < compare[1]:
compare[0] = i
compare[1] = len(route_list.get(i))
else:
for i in route_list:
compare[0] = i
compare[1] = len(route_list.get(i))
post_compare = []
print("TEST - Mid route list: ", mid_route_list)
if len(mid_route_list) != 0:
for i in mid_route_list:
post_compare.append(i)
route_list[i] = mid_route_list.get(i)
post_compare.append(compare[0])
else:
post_compare.append(compare[0])
"""
Upon comparison, it will start capturing the nodes within the bus path and store in plot_list
"""
plot_list = []
try:
print("TEST - post_Compare: ", post_compare)
print("TEST - Route list: ", route_list)
for count in range(0, len(post_compare)):
for x in route_list.get(str(post_compare[count])):
for i in G.nodes:
if str(G.nodes.get(i).get('asset_ref')) == str(x):
plot_list.append(G.nodes.get(i).get('osmid'))
break
except:
return -1
edge_list = []
punggol = (1.403948, 103.909048)
"""
It will generate out the list of node ID for the UI to plot
"""
a = ox.load_graphml('Bus_graph.graphml')
for x in plot_list:
edge_list.append(get_nearestedge_node(x, a, G))
print("TEST - Plot list: ", plot_list)
print("TEST - Edge list: ", edge_list)
final_route_list = []
count_stops = len(plot_list)
for x in range(0, len(edge_list) - 1):
final_route_list.append(
nx.shortest_path(a, edge_list[x], edge_list[x + 1]))
print(final_route_list)
return final_route_list