def speed_on_the_ground(graph: nx.DiGraph, filename: str):
mpl.use("Agg")
with commons.Section("Getting the background OSM map"):
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, name="pos"),
index=["lon", "lat"]).T
extent = maps.ax4(nodes.lat, nodes.lon)
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent))
with mpl.rc_context(PARAM['mpl_style']), commons.Axes() as ax1:
velocity = pd.Series(nx.get_edge_attributes(
graph, name="len")) / pd.Series(
nx.get_edge_attributes(graph, name=PARAM['edge_time_attr']))
cmap = LinearSegmentedColormap.from_list(
name="noname", colors=["brown", "r", "orange", "g"])
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
nx.draw(graph,
ax=ax1,
pos=nx.get_node_attributes(graph, name="pos"),
edgelist=list(velocity.index),
edge_color=list(velocity),
edge_cmap=cmap,
edge_vmin=0,
edge_vmax=7,
with_labels=False,
arrows=False,
node_size=0,
alpha=0.8,
width=0.3)
fn = PARAM['out_images_path'] / commons.myname() / F"{filename}.png"
ax1.figure.savefig(commons.makedirs(fn))
def nx_draw_met_by_len(graph, edges_met=None, mpl_backend="Agg", printer=None):
mpl.use(mpl_backend)
with Section("Preparing to draw graph", out=printer):
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, name="loc"),
index=["lat", "lon"]).T
edges_len = pd.Series(data=nx.get_edge_attributes(graph, name="len"),
name="len")
if edges_met is not None:
edges_met = pd.Series(name="met", data=edges_met)
else:
edges_met = pd.Series(name="met",
data=nx.get_edge_attributes(graph,
name="met"))
cmap = LinearSegmentedColormap.from_list(
name="noname", colors=["g", "y", "r", "brown"])
with Section("Getting the background OSM map", out=printer):
extent = maps.ax4(nodes.lat, nodes.lon)
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent))
with Section("Drawing image", out=printer):
fig: plt.Figure
ax1: plt.Axes
(fig, ax1) = plt.subplots()
# The background map
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
ax1.axis("off")
edge_colors = (edges_met / edges_len).clip(lower=0.9, upper=1.5)
nx.draw(graph,
ax=ax1,
pos=nx.get_node_attributes(graph, name="pos"),
edge_list=list(edge_colors.index),
edge_color=list(edge_colors),
edge_cmap=cmap,
with_labels=False,
arrows=False,
node_size=0,
alpha=1,
width=0.4)
ax1.set_xlim(extent[0:2])
ax1.set_ylim(extent[2:4])
try:
# Note:
# "yield (fig, ax1)" does not work with the "retry" context manager
return iter([(fig, ax1)])
finally:
plt.close(fig)
def compare_multiple_trajectories(table_name):
mpl.use("Agg")
# Number of trips to plot
N = 10
# Number of trajectories per trip
M = 12
graph = get_road_graph()
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, "loc"), index=["lat", "lon"]).T
edges_len = nx.get_edge_attributes(graph, name="len")
where = "('2016-05-02 08:00' <= pickup_datetime) and (pickup_datetime <= '2016-05-02 09:00')"
trips = get_trip_data(table_name, graph, order="", where=where)
trips = trips.sample(min(N, len(trips)), random_state=1)
logger.debug(F"{len(trips)} trips")
with Section("Getting the background OSM map", out=logger.debug):
extent = maps.ax4(nodes.lat, nodes.lon)
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent))
with plt.style.context({**PARAM['mpl_style'], 'font.size': 5}), Axes() as ax1:
# The background map
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
ax1.axis("off")
ax1.set_xlim(extent[0:2])
ax1.set_ylim(extent[2:4])
for (__, trip) in trips.iterrows():
with Section("Computing candidate trajectories", out=logger.debug):
trajectories = pd.DataFrame(data={'path': [
path
for (__, path) in
zip(range(M), nx.shortest_simple_paths(graph, source=trip.u, target=trip.v))
]})
trajectories['dist'] = [sum(edges_len[e] for e in pairwise(path)) for path in trajectories.path]
trajectories = trajectories.sort_values(by='dist', ascending=False)
marker = dict(markersize=2, markeredgewidth=0.2, markerfacecolor="None")
ax1.plot(trip['pickup_longitude'], trip['pickup_latitude'], 'og', **marker)
ax1.plot(trip['dropoff_longitude'], trip['dropoff_latitude'], 'xr', **marker)
cmap = LinearSegmentedColormap.from_list(name="noname", colors=["g", "orange", "r", "brown"])
colors = cmap(pd.Series(trajectories['dist'] / trip['distance']).rank(pct=True))
for (c, path) in zip(colors, trajectories.path):
(y, x) = nodes.loc[list(path)].values.T
ax1.plot(x, y, c=c, alpha=0.5, lw=0.3)
# Save to file
fn = os.path.join(PARAM['out_images_path'], F"{myname()}/{table_name}.png")
ax1.figure.savefig(makedirs(fn))
def trip_trajectories_ingraph(table_name):
mpl.use("Agg")
# Max number of trajectories to plot
N = 1000
graph = get_road_graph()
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, "loc"), index=["lat", "lon"]).T
trips = get_trip_data(table_name, graph)
trips = trips.sample(min(N, len(trips)))
logger.debug(F"{len(trips)} trips")
logger.debug("Computing trajectories")
trajectories = parallel_map(GraphPathDist(graph).path_only, zip(trips.u, trips.v))
with Section("Getting the background OSM map", out=logger.debug):
extent = maps.ax4(nodes.lat, nodes.lon)
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent))
with plt.style.context({**PARAM['mpl_style'], 'font.size': 5}):
with Axes() as ax1:
# The background map
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
ax1.axis("off")
ax1.set_xlim(extent[0:2])
ax1.set_ylim(extent[2:4])
c = 'b'
if ("green" in table_name): c = "green"
if ("yello" in table_name): c = "orange"
logger.debug("Plotting trajectories")
for traj in trajectories:
(y, x) = nodes.loc[list(traj)].values.T
ax1.plot(x, y, c=c, alpha=0.1, lw=0.3)
# Save to file
fn = os.path.join(PARAM['out_images_path'], F"{myname()}/{table_name}.png")
ax1.figure.savefig(makedirs(fn))
# Meta info
json.dump({'number_of_trajectories': len(trips)}, open((fn + ".txt"), 'w'))
def trip_trajectories_initial(table_name):
mpl.use("Agg")
N = 10000
cols = [
"pickup_latitude", "pickup_longitude", "dropoff_latitude",
"dropoff_longitude"
]
sql = F"SELECT {(', '.join(cols))} FROM [{table_name}] ORDER BY RANDOM() LIMIT {N}"
df = query(sql)
df = df.rename(columns=dict(zip(cols, ["lat0", "lon0", "lat1", "lon1"])))
fig: plt.Figure
ax1: plt.Axes
(fig, ax1) = plt.subplots()
ax1.tick_params(axis='both', which='both', labelsize='3')
c = "b"
if ("green" in table_name): c = "green"
if ("yellow" in table_name): c = "orange"
for (yy, xx) in zip(df[['lat0', 'lat1']].values, df[['lon0',
'lon1']].values):
ax1.plot(xx, yy, c=c, ls='-', alpha=0.1, lw=0.1)
ax1.axis("off")
# Get the background map
axis = ax1.axis()
img_map = maps.get_map_by_bbox(maps.ax2mb(*axis))
ax1.imshow(img_map, extent=axis, interpolation='quadric', zorder=-100)
fn = os.path.join(PARAM['out_images_path'], F"{myname()}/{table_name}.png")
fig.savefig(makedirs(fn), **PARAM['savefig_args'])
def trip_trajectories_velocity(table_name):
mpl.use("Agg")
# Max number of trajectories to use
N = 10000
graph = get_road_graph()
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, "loc"), index=["lat", "lon"]).T
edge_name = pd.Series(nx.get_edge_attributes(graph, name="name"))
where = "('2016-05-02 08:00' <= pickup_datetime) and (pickup_datetime <= '2016-05-02 09:00')"
trips = get_trip_data(table_name, graph, order="", limit=N, where=where)
trips['velocity'] = trips['distance'] / trips['duration/s']
trips = trips.sort_values(by='velocity', ascending=True)
logger.debug(F"{len(trips)} trips")
with Section("Computing estimated trajectories", out=logger.debug):
trips['traj'] = parallel_map(GraphPathDist(graph).path_only, zip(trips.u, trips.v))
with Section("Getting the background OSM map", out=logger.debug):
extent = maps.ax4(nodes.lat, nodes.lon)
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent))
with Section("Computing edge velocities", out=logger.debug):
edge_vel = defaultdict(list)
for (traj, v) in zip(trips.traj, trips.velocity):
for e in pairwise(traj):
edge_vel[e].append(v)
edge_vel = pd.Series({e: np.mean(v or np.nan) for (e, v) in edge_vel.items()}, index=graph.edges)
edge_vel = edge_vel.dropna()
with plt.style.context({**PARAM['mpl_style'], 'font.size': 5}), Axes() as ax1:
# The background map
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
ax1.axis("off")
ax1.set_xlim(extent[0:2])
ax1.set_ylim(extent[2:4])
cmap_velocity = LinearSegmentedColormap.from_list(name="noname", colors=["brown", "r", "orange", "g"])
# marker = dict(markersize=0.5, markeredgewidth=0.1, markerfacecolor="None")
# ax1.plot(trips['pickup_longitude'], trips['pickup_latitude'], 'og', **marker)
# ax1.plot(trips['dropoff_longitude'], trips['dropoff_latitude'], 'xr', **marker)
# for e in edge_name[edge_name == "65th Street Transverse"].index:
# print(e, edge_vel[e])
edge_vel: pd.Series
# edge_vel = edge_vel.rank(pct=True)
edge_vel = edge_vel.clip(lower=2, upper=6).round()
edge_vel = (edge_vel - edge_vel.min()) / (edge_vel.max() - edge_vel.min())
edge_vel = edge_vel.apply(cmap_velocity)
nx.draw_networkx_edges(
graph.edge_subgraph(edge_vel.index),
ax=ax1,
pos=nx.get_node_attributes(graph, name="pos"),
edge_list=list(edge_vel.index),
edge_color=list(edge_vel),
# edge_cmap=cmap_velocity,
# vmin=0, vmax=1,
with_labels=False, arrows=False, node_size=0, alpha=0.8, width=0.3,
)
# Save to file
fn = os.path.join(PARAM['out_images_path'], F"{myname()}/{table_name}.png")
ax1.figure.savefig(makedirs(fn))
# Meta info
json.dump({'number_of_trajectories': len(trips)}, open((fn + ".txt"), 'w'))
def mm_callback(result):
if (result['status'] == "zero"):
print("(Preparing)")
if (result['status'] == "init"):
print("(Optimizing)")
if (result['status'] == "opti"):
if (dt.datetime.now() < result.get('nfu', dt.datetime.min)):
return
if (result['status'] == "done"):
print("(Done)")
if (result['status'] == "zero"):
fig: plt.Figure
ax: plt.Axes
(fig, ax) = plt.subplots()
for (n, (y, x)) in enumerate(result['waypoints']):
ax.plot(x, y, 'bo')
ax.axis(commons.niceaxis(ax.axis(), expand=1.1))
ax.autoscale(enable=False)
# Download the background map
try:
mapi = maps.get_map_by_bbox(maps.ax2mb(*ax.axis()),
token=mapbox_token)
except maps.URLError:
commons.logger.warning("No map (no connection?)")
mapi = None
result['plt'] = {
'fig': fig,
'ax': ax,
'map': mapi,
'bbox': ax.axis()
}
if (result['status'] in ["zero", "opti", "done"]):
fig: plt.Figure
ax: plt.Axes
(fig, ax, mapi,
bbox) = commons.inspect({'plt':
('fig', 'ax', 'map', 'bbox')})(result)
# Clear the axes
ax.cla()
# Apply the background map
if mapi:
img = ax.imshow(mapi,
extent=bbox,
interpolation='none',
zorder=-100)
for (n, (y, x)) in enumerate(result['waypoints']):
ax.plot(x, y, 'o', c='m', markersize=4)
if ('geo_path' in result):
(y, x) = zip(*result['geo_path'])
ax.plot(x, y, 'b--', linewidth=2, zorder=-50)
if ('(edge_clouds)' in result):
for (nc, pc) in enumerate(result['(edge_clouds)']):
for (e, p) in pc.items():
(y, x) = zip(*[g.nodes[i]['pos'] for i in e])
c = ('g' if
(result['(active_edges)'][nc] == e) else 'r')
ax.plot(x,
y,
'-',
c=c,
linewidth=2,
alpha=(p / max(pc.values())),
zorder=150)
ax.axis(bbox)
plt.pause(0.1)
if (result['status'] == "opti"):
# Next figure update
result['nfu'] = dt.datetime.now() + dt.timedelta(seconds=4)
if (result['status'] == "done"):
open("graph_mm_callback.gpx", 'w').write(
simple_gpx(result['waypoints'], [result['geo_path']]).to_xml())
def vis1() :
# OSM = pickle.load(open(IFILE['OSM'], 'rb'))
# for (route_id, route) in OSM['rels']['route'].items():
# # Skip non-bus routes
# if not (route['t'].get('route') == 'bus'): continue
#
# route_name = route['t'].get('name')
#
# route_ref = route['t']['ref']
# #if (route_ref == '88') :
# print(route_name, route_id, route['t'])
# exit(39)
routeid_of = (lambda r: r['SubRouteUID'])
# List of filenames, one file per physical bus, identified by plate number
bus_files = commons.ls(IFILE['busses'].format(busid="*"))
# Refile bus runs by their route ID
runs_by_route = defaultdict(list)
for fn in bus_files :
runs = commons.zipjson_load(fn)
for run in runs :
runs_by_route[routeid_of(run)].append(run)
#
route_stops = commons.index_dicts_by_key(commons.zipjson_load(IFILE['route-stops']), routeid_of)
# Are those valid route ID that can be found among the routes?
unknown_route_ids = sorted(set(runs_by_route.keys()) - set(route_stops.keys()))
if unknown_route_ids :
print("The following route IDs from bus records are unknown:")
print(", ".join(unknown_route_ids))
raise KeyError("Unkown route IDs in bus records")
#
route_uid = 'KHH24'
runs = runs_by_route[route_uid]
route = route_stops[route_uid]
# Kaohsiung (left, bottom, right, top)
bbox = (120.2593, 22.5828, 120.3935, 22.6886)
(left, bottom, right, top) = bbox
# Download the background map
i = maps.get_map_by_bbox(bbox, token=PARAM['mapbox_api_token'])
# Show the background map
(fig, ax) = plt.subplots()
plt.ion()
ax.axis([left, right, bottom, top])
ax.imshow(i, extent=(left, right, bottom, top), interpolation='quadric')
#fig.canvas.draw_idle()
plt.pause(0.1)
stops_by_direction = dict(zip(route['Direction'], route['Stops']))
# Draw stops for both route directions
for (dir, stops) in stops_by_direction.items() :
# Stop locations
(y, x) = zip(*[
commons.inspect({'StopPosition': ('PositionLat', 'PositionLon')})(stop)
for stop in stops
])
# Plot as dots
ax.scatter(x, y, c=('b' if dir else 'g'), marker='o', s=4)
# Show bus location
for run in runs :
# Trace bus
(y, x) = (run['PositionLat'], run['PositionLon'])
h1 = ax.plot(x, y, '--+', c='r', linewidth=1)
h2 = ax.plot(x[0], y[0], 'o', c='r')
h3 = ax.plot(x[-1], y[-1], 's', c='r')
plt.title(run['PlateNumb'])
#plt.savefig("{}.png".format(route_uid), dpi=180)
plt.pause(0.1)
bus_at_stops(run, stops_by_direction[run['Direction']])
plt.pause(0.1)
[h[0].remove() for h in [h1, h2, h3]]
return
def nx_draw(graph, nodes=None, edges=None):
# mpl.use("Agg")
logger.debug("Preparing to draw graph")
if nodes is None:
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, name="loc"),
index=["lat", "lon"]).T
if edges is None:
edges = pd.DataFrame([
pd.Series(data=nx.get_edge_attributes(graph, name="met"),
name="met"),
pd.Series(data=nx.get_edge_attributes(graph, name="len"),
name="len"),
]).T
edges['color'] = (edges.met / edges.len)
cmap = LinearSegmentedColormap.from_list(name="noname",
colors=["g", "y", "r", "brown"])
# Axes window
# extent = np.dot(
# [[min(nodes.lon), max(nodes.lon)], [min(nodes.lat), max(nodes.lat)]],
# (lambda s: np.asarray([[1 + s, -s], [-s, 1 + s]]))(0.01)
# ).flatten()
# Manhattan south
extent = [-74, -73.96, 40.73, 40.78]
logger.debug("Getting the background OSM map")
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent),
style=maps.MapBoxStyle.streets)
fig: plt.Figure
ax1: plt.Axes
(fig, ax1) = plt.subplots()
logger.debug("Drawing image")
# The background map
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
ax1.axis("off")
edges.color[edges.color > 1.5] = 1.5
edges.color[edges.color < 0.9] = 0.9
g = nx.draw_networkx_edges(graph,
ax=ax1,
pos=nx.get_node_attributes(graph, name="pos"),
edge_list=list(edges.index),
edge_color=list(edges.color),
edge_cmap=cmap,
with_labels=False,
arrows=False,
node_size=0,
alpha=1,
width=2)
# https://stackoverflow.com/questions/26739248/how-to-add-a-simple-colorbar-to-a-network-graph-plot-in-python
(vmin, vmax) = (np.min(edges.color), np.max(edges.color))
sm = plt.cm.ScalarMappable(cmap=cmap,
norm=plt.Normalize(vmin=vmin, vmax=vmax))
sm._A = []
plt.colorbar(sm)
# https://networkx.github.io/documentation/stable/auto_examples/drawing/plot_directed.html
# pc = mpl.collections.PatchCollection(g, cmap=cmap)
# pc.set_array(list(edges.color))
# plt.colorbar(pc)
# cax = fig.add_axes([ax1.get_position().x1 + 0.01, ax1.get_position().y0, 0.02, ax1.get_position().height])
# cbar = fig.colorbar(im, cax=cax)
ax1.set_xlim(extent[0:2])
ax1.set_ylim(extent[2:4])
logger.debug("OK")
try:
yield (fig, ax1)
finally:
plt.close(fig)
def make_transit_img(J, backend='Agg') -> bytes:
import matplotlib as mpl
mpl.use(backend)
mpl.rcParams['figure.max_open_warning'] = 10
import dateutil.parser as dateparser
import matplotlib.pyplot as plt
ax: plt.Axes
(fig, ax) = plt.subplots()
origin = {
'x': J['origin']['x'],
't': dateparser.parse(J['origin']['t']),
'desc': J['origin'].get('desc'),
}
# Location --> Transit time in minutes ; keep track of duplicates
gohere = commons.index_dicts_by_key(J['gohere'],
key_func=(lambda __: tuple(__['x'])),
collapse_repetitive=False)
# Keep only the minimal reach time, convert to minutes
gohere = {x: (min(attr['s']) / 60) for (x, attr) in gohere.items()}
# # Cut-off (and normalize)
T = 60 # Minutes
# gohere = { p : s for (p, s) in gohere.items() if (s <= T) }
# Reindex datapoints by (x, y) pairs
contour_pts = dict(zip(map(ll2xy, gohere.keys()), gohere.values()))
#boxes = dict(boxify(gohere, maxinbox=10))
# "Inner" Kaohsiung
bbox = (120.2593, 22.5828, 120.3935, 22.6886)
# Set plot view to the bbox
ax.axis(maps.mb2ax(*bbox))
ax.autoscale(enable=False)
try:
background_map = maps.get_map_by_bbox(bbox, **PARAM['mapbox'])
ax.imshow(background_map,
interpolation='quadric',
extent=maps.mb2ax(*bbox),
zorder=-100)
except Exception as e:
commons.logger.warning("No background map ({})".format(e))
# Cross at the origin
((ox, oy), ot) = (ll2xy(origin['x']),
origin['t'].strftime("%Y-%m-%d / %H:%M"))
ax.plot(ox, oy, 'gx', ms=3, mew=0.2)
ax.text(ox, oy, s="\n{}".format(ot), ha='center', va='top', fontsize=2)
# Show all datapoints
#ax.scatter(*zip(*contour_pts), marker='o', c='k', s=0.1, lw=0, edgecolor='none')
# # Hack! for corners
# for (x, y) in product(ax.axis()[:2], ax.axis()[2:]) :
# contour_pts[(x, y)] = max(gohere.values())
cmap = plt.get_cmap('Purples')
cmap.set_over('k')
# https://stackoverflow.com/questions/37327308/add-alpha-to-an-existing-matplotlib-colormap
from matplotlib.colors import ListedColormap
cmap = ListedColormap(
np.vstack(
[cmap(np.arange(cmap.N))[:, 0:3].T,
np.linspace(0, 0.5, cmap.N)]).T)
(x, y) = zip(*contour_pts)
levels = list(range(0, T, 5))
c = ax.tricontourf(x,
y,
list(contour_pts.values()),
levels=levels,
zorder=100,
cmap=cmap,
extent=maps.mb2ax(*bbox),
extend='max')
# Reduce fontsize
ax.tick_params(axis='both', which='both', labelsize='xx-small')
fig.colorbar(c).ax.tick_params(axis='both',
which='both',
labelsize='xx-small')
# import matplotlib.patches as patches
# boxes = dict(boxify(contour_pts, maxinbox=20))
# for (bb, gohere_part) in list(boxes.items()) :
# (y, x) = bb[0:2]
# (h, w) = (bb[2]-bb[0], bb[3]-bb[1])
# ax.add_patch(patches.Rectangle((x, y), w, h, linewidth=0.5, edgecolor='k', facecolor='none'))
# # for (latlon, s) in list(gohere_part.items()) :
# # ax.plot(*ll2xy(latlon), 'o', c=plt.get_cmap('Purples')(s), markersize=3)
buffer = io.BytesIO()
fig.savefig(buffer, bbox_inches='tight', pad_inches=0, dpi=300)
buffer.seek(0)
if backend.lower() in ["tkagg"]:
plt.ion()
plt.show()
plt.pause(0.1)
plt.close(fig)
return buffer.read()