def nx_draw_met_by_len(graph, edges_met=None, mpl_backend="Agg", printer=None):
mpl.use(mpl_backend)
with Section("Preparing to draw graph", out=printer):
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, name="loc"),
index=["lat", "lon"]).T
edges_len = pd.Series(data=nx.get_edge_attributes(graph, name="len"),
name="len")
if edges_met is not None:
edges_met = pd.Series(name="met", data=edges_met)
else:
edges_met = pd.Series(name="met",
data=nx.get_edge_attributes(graph,
name="met"))
cmap = LinearSegmentedColormap.from_list(
name="noname", colors=["g", "y", "r", "brown"])
with Section("Getting the background OSM map", out=printer):
extent = maps.ax4(nodes.lat, nodes.lon)
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent))
with Section("Drawing image", out=printer):
fig: plt.Figure
ax1: plt.Axes
(fig, ax1) = plt.subplots()
# The background map
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
ax1.axis("off")
edge_colors = (edges_met / edges_len).clip(lower=0.9, upper=1.5)
nx.draw(graph,
ax=ax1,
pos=nx.get_node_attributes(graph, name="pos"),
edge_list=list(edge_colors.index),
edge_color=list(edge_colors),
edge_cmap=cmap,
with_labels=False,
arrows=False,
node_size=0,
alpha=1,
width=0.4)
ax1.set_xlim(extent[0:2])
ax1.set_ylim(extent[2:4])
try:
# Note:
# "yield (fig, ax1)" does not work with the "retry" context manager
return iter([(fig, ax1)])
finally:
plt.close(fig)
def compare_multiple_trajectories(table_name):
mpl.use("Agg")
# Number of trips to plot
N = 10
# Number of trajectories per trip
M = 12
graph = get_road_graph()
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, "loc"), index=["lat", "lon"]).T
edges_len = nx.get_edge_attributes(graph, name="len")
where = "('2016-05-02 08:00' <= pickup_datetime) and (pickup_datetime <= '2016-05-02 09:00')"
trips = get_trip_data(table_name, graph, order="", where=where)
trips = trips.sample(min(N, len(trips)), random_state=1)
logger.debug(F"{len(trips)} trips")
with Section("Getting the background OSM map", out=logger.debug):
extent = maps.ax4(nodes.lat, nodes.lon)
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent))
with plt.style.context({**PARAM['mpl_style'], 'font.size': 5}), Axes() as ax1:
# The background map
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
ax1.axis("off")
ax1.set_xlim(extent[0:2])
ax1.set_ylim(extent[2:4])
for (__, trip) in trips.iterrows():
with Section("Computing candidate trajectories", out=logger.debug):
trajectories = pd.DataFrame(data={'path': [
path
for (__, path) in
zip(range(M), nx.shortest_simple_paths(graph, source=trip.u, target=trip.v))
]})
trajectories['dist'] = [sum(edges_len[e] for e in pairwise(path)) for path in trajectories.path]
trajectories = trajectories.sort_values(by='dist', ascending=False)
marker = dict(markersize=2, markeredgewidth=0.2, markerfacecolor="None")
ax1.plot(trip['pickup_longitude'], trip['pickup_latitude'], 'og', **marker)
ax1.plot(trip['dropoff_longitude'], trip['dropoff_latitude'], 'xr', **marker)
cmap = LinearSegmentedColormap.from_list(name="noname", colors=["g", "orange", "r", "brown"])
colors = cmap(pd.Series(trajectories['dist'] / trip['distance']).rank(pct=True))
for (c, path) in zip(colors, trajectories.path):
(y, x) = nodes.loc[list(path)].values.T
ax1.plot(x, y, c=c, alpha=0.5, lw=0.3)
# Save to file
fn = os.path.join(PARAM['out_images_path'], F"{myname()}/{table_name}.png")
ax1.figure.savefig(makedirs(fn))
def main():
with Section("Plotting past results", out=logger.info):
try:
logger.debug("Press CTRL+C to skip and rerun the experiments instead...")
sleep(5)
plot_results()
return
except KeyboardInterrupt:
pass
with Section("Running new set of experiments", out=logger.info):
sleep(4)
run_experiments()
def trip_distance_vs_shortest(table_name):
mpl.use("Agg")
graph = get_road_graph()
trips = get_trip_data(table_name, graph)
with Section("Computing shortest distances", out=logger.debug):
trips = trips.join(
pd.DataFrame(
data=parallel_map(GraphPathDist(graph, edge_weight="len"), zip(trips.u, trips.v)),
columns=['path', 'shortest'], index=trips.index,
)
)
# On-graph distance vs reported distance [meters]
df: pd.DataFrame
df = pd.DataFrame(data=dict(
reported=(trips['distance']),
shortest=(trips['shortest']),
))
# Convert to [km] and stay below 10km
df = df.applymap(lambda x: (x / 1e3))
df = df.applymap(lambda km: (km if (km < 10) else np.nan)).dropna()
# Hour of the day
df['h'] = trips['pickup_datetime'].dt.hour
with plt.style.context(PARAM['mpl_style']):
with Axes() as ax1:
ax1.set_aspect(aspect="equal", adjustable="box")
ax1.grid()
ax1.plot(*(2 * [[0, df[['reported', 'shortest']].values.max()]]), c='k', ls='--', lw=0.5, zorder=100)
for (h, hdf) in df.groupby(df['h']):
c = plt.get_cmap("twilight_shifted")([h / 24])
ax1.scatter(
hdf['reported'], hdf['shortest'],
c=c, s=3, alpha=0.8, lw=0, zorder=10,
label=(F"{len(hdf)} trips at {h}h")
)
ax1.set_xlabel("Reported distance, km")
ax1.set_ylabel("Naive graph distance, km")
ax1.set_xticks(range(11))
ax1.set_yticks(range(11))
ax1.legend()
# Save to file
fn = os.path.join(PARAM['out_images_path'], F"{myname()}/{table_name}.png")
ax1.figure.savefig(makedirs(fn))
# Meta info
json.dump({'number_of_datapoints': len(df)}, open((fn + ".txt"), 'w'))
def get_trip_data(table_name, graph, where="", order="random()", limit=100000) -> pd.DataFrame:
# Load taxi trips from the database
with Section("Reading the database", out=logger.debug):
# sql = F"SELECT * FROM [{table_name}] ORDER BY RANDOM() LIMIT 1000" # DEBUG
# sql = F"SELECT * FROM [{table_name}] ORDER BY RANDOM() LIMIT 10000" # DEBUG
sql = F"SELECT * FROM [{table_name}]"
sql += (F" WHERE ({where}) " if where else "")
sql += (F" ORDER BY ({order}) " if order else "")
sql += (F" LIMIT ({limit}) " if limit else "")
trips = pd.read_sql_query(
sql=sql,
con=sqlite3.connect(PARAM['taxidata']),
parse_dates=["pickup_datetime", "dropoff_datetime"],
)
# Trip duration
trips['duration/s'] = (trips['dropoff_datetime'] - trips['pickup_datetime']).dt.total_seconds()
with Section("Computing nearest in-graph nodes", out=logger.debug):
# Nearest-node computer
nearest_node = GraphNearestNode(graph)
# (index, values) correspond to (graph node id, distance)
U = nearest_node(list(zip(trips['pickup_latitude'], trips['pickup_longitude'])))
V = nearest_node(list(zip(trips['dropoff_latitude'], trips['dropoff_longitude'])))
# In-graph node estimates of pickup and dropoff
trips['u'] = U.index
trips['v'] = V.index
# Grace distance from given lat/lon to nearest in-graph lat/lon
MAX_NEAREST = 20 # meters
trips = trips.loc[(U.values <= MAX_NEAREST) & (V.values <= MAX_NEAREST)]
return trips
def trip_trajectories_ingraph(table_name):
mpl.use("Agg")
# Max number of trajectories to plot
N = 1000
graph = get_road_graph()
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, "loc"), index=["lat", "lon"]).T
trips = get_trip_data(table_name, graph)
trips = trips.sample(min(N, len(trips)))
logger.debug(F"{len(trips)} trips")
logger.debug("Computing trajectories")
trajectories = parallel_map(GraphPathDist(graph).path_only, zip(trips.u, trips.v))
with Section("Getting the background OSM map", out=logger.debug):
extent = maps.ax4(nodes.lat, nodes.lon)
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent))
with plt.style.context({**PARAM['mpl_style'], 'font.size': 5}):
with Axes() as ax1:
# The background map
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
ax1.axis("off")
ax1.set_xlim(extent[0:2])
ax1.set_ylim(extent[2:4])
c = 'b'
if ("green" in table_name): c = "green"
if ("yello" in table_name): c = "orange"
logger.debug("Plotting trajectories")
for traj in trajectories:
(y, x) = nodes.loc[list(traj)].values.T
ax1.plot(x, y, c=c, alpha=0.1, lw=0.3)
# Save to file
fn = os.path.join(PARAM['out_images_path'], F"{myname()}/{table_name}.png")
ax1.figure.savefig(makedirs(fn))
# Meta info
json.dump({'number_of_trajectories': len(trips)}, open((fn + ".txt"), 'w'))
def run_experiments() -> pd.DataFrame:
aliquot = datetime.utcnow().strftime("%Y%m%d-%H%M%S")
setups = pd.DataFrame(
# data=list(product([32, 64], [0.1, 0.2, 0.4, 0.8], [(2 ** n) for n in range[7, 13]])),
data=list(product([32, 64], [0.1, 0.2, 0.4, 0.8], [(2 ** n) for n in range[7, 10]])),
# data=list(product([64], [0.5], [2 ** 13])),
# data=list(product([(2 ** n) for n in range[2, 6]], [0.1, 0.2, 0.4], [100, 1000, 10000])),
# data=list(product([4, 8, 16], [0.1], [10, 100])),
columns=["graph_size", "noise", "ntrips"],
)
logger.debug('\n'.join(map(str, ["Experiments:", setups])))
for setup in setups.itertuples(index=False):
# Preserve datatypes
setup = dict(setup._asdict())
# Alternative:
# setup = setup.astype({'graph_size': int, 'noise': float, 'ntrips': int})
with Section(F"Experiment {setup} is on", out=logger.info):
# https://www.dataquest.io/blog/settingwithcopywarning/
with pd.option_context('mode.chained_assignment', None):
history = experiment(**setup, num_rounds=64)
with open(makedirs(PARAM['out_experiment_results'].format(aliquot=aliquot, ext="pkl")), 'ab') as fd:
pickle.dump({**setup, 'history': history}, fd)
# results.to_json(makedirs(PARAM['out_experiment_results'].format(aliquot=aliquot, ext="json")))
with open(makedirs(PARAM['out_experiment_results'].format(aliquot=aliquot, ext="json")), 'w') as fd:
json.dump(
{
'setups': setups.to_json(),
'script': this_module_body(),
'timestamp': datetime.now(tz=timezone.utc).isoformat(),
},
fd
)
return setups
def get_road_graph() -> nx.DiGraph:
with Section("Loading the road graph", out=logger.debug):
return largest_component(pickle.load(open(PARAM['road_graph'], 'rb')))
def trip_trajectories_velocity(table_name):
mpl.use("Agg")
# Max number of trajectories to use
N = 10000
graph = get_road_graph()
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, "loc"), index=["lat", "lon"]).T
edge_name = pd.Series(nx.get_edge_attributes(graph, name="name"))
where = "('2016-05-02 08:00' <= pickup_datetime) and (pickup_datetime <= '2016-05-02 09:00')"
trips = get_trip_data(table_name, graph, order="", limit=N, where=where)
trips['velocity'] = trips['distance'] / trips['duration/s']
trips = trips.sort_values(by='velocity', ascending=True)
logger.debug(F"{len(trips)} trips")
with Section("Computing estimated trajectories", out=logger.debug):
trips['traj'] = parallel_map(GraphPathDist(graph).path_only, zip(trips.u, trips.v))
with Section("Getting the background OSM map", out=logger.debug):
extent = maps.ax4(nodes.lat, nodes.lon)
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent))
with Section("Computing edge velocities", out=logger.debug):
edge_vel = defaultdict(list)
for (traj, v) in zip(trips.traj, trips.velocity):
for e in pairwise(traj):
edge_vel[e].append(v)
edge_vel = pd.Series({e: np.mean(v or np.nan) for (e, v) in edge_vel.items()}, index=graph.edges)
edge_vel = edge_vel.dropna()
with plt.style.context({**PARAM['mpl_style'], 'font.size': 5}), Axes() as ax1:
# The background map
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
ax1.axis("off")
ax1.set_xlim(extent[0:2])
ax1.set_ylim(extent[2:4])
cmap_velocity = LinearSegmentedColormap.from_list(name="noname", colors=["brown", "r", "orange", "g"])
# marker = dict(markersize=0.5, markeredgewidth=0.1, markerfacecolor="None")
# ax1.plot(trips['pickup_longitude'], trips['pickup_latitude'], 'og', **marker)
# ax1.plot(trips['dropoff_longitude'], trips['dropoff_latitude'], 'xr', **marker)
# for e in edge_name[edge_name == "65th Street Transverse"].index:
# print(e, edge_vel[e])
edge_vel: pd.Series
# edge_vel = edge_vel.rank(pct=True)
edge_vel = edge_vel.clip(lower=2, upper=6).round()
edge_vel = (edge_vel - edge_vel.min()) / (edge_vel.max() - edge_vel.min())
edge_vel = edge_vel.apply(cmap_velocity)
nx.draw_networkx_edges(
graph.edge_subgraph(edge_vel.index),
ax=ax1,
pos=nx.get_node_attributes(graph, name="pos"),
edge_list=list(edge_vel.index),
edge_color=list(edge_vel),
# edge_cmap=cmap_velocity,
# vmin=0, vmax=1,
with_labels=False, arrows=False, node_size=0, alpha=0.8, width=0.3,
)
# Save to file
fn = os.path.join(PARAM['out_images_path'], F"{myname()}/{table_name}.png")
ax1.figure.savefig(makedirs(fn))
# Meta info
json.dump({'number_of_trajectories': len(trips)}, open((fn + ".txt"), 'w'))
g = odd_king_graph(128, 128)
nodes = pd.DataFrame(data=nx.get_node_attributes(g, name="pos"),
index=["lon", "lat"]).T
N = 1000
node_loc = nodes.to_numpy()
def d(u, v):
(ux, uy) = node_loc[u]
(vx, vy) = node_loc[v]
return 3 * ((ux - vx)**2 + (uy - vy)**2)**(1 / 2)
with Section("astar_path", out=print):
for n in range(N):
(u, v) = nodes.sample(2).index
nx.astar_path(g, source=u, target=v, heuristic=d, weight="len")
with Section("shortest_path (dijkstra)", out=print):
for n in range(N):
(u, v) = nodes.sample(2).index
nx.shortest_path(g,
source=u,
target=v,
weight="len",
method="dijkstra")
with Section("shortest_path (bellman-ford)", out=print):
for n in range(N):
"oneway",
"foot",
"pedestrian",
"turn",
],
'max_graph_edge_len':
20,
'out_road_graph':
makedirs("data/road_graph/UV/nx_digraph_naive.pkl"),
'out_road_graph_sketch':
makedirs("data/road_graph/sketch.png"),
'savefig_args':
dict(bbox_inches='tight', pad_inches=0, dpi=300),
}
with Section("Loading OSM archive", out=print):
with ZipFile(PARAM['osm_archive'], mode='r') as archive:
J = {
name: json.load(archive.open("data"))
for name in archive.namelist()
}
assert (1 == len(J))
J = next(iter(J.values()))
J = J['elements']
# OSM nodes and OSM ways as DataFrame
nodes: pd.DataFrame
ways: pd.DataFrame
(nodes, ways) = [
pd.DataFrame(data=(x for x in J if (x['type'] == t))).set_index(
from mpl_toolkits import mplot3d
import matplotlib.dates as mdates
import matplotlib.pyplot as plt
import seaborn as sb
PARAM = {
'road_graph':
"../data_preparation/data/road_graph/UV/nx_digraph_naive.pkl",
# 'taxidata': "../data_preparation/data/taxidata/sqlite/UV/db.db",
'edges_met':
"../models/manhattan_metric/yellow_tripdata_2016-05/1/08/edges_met.pkl",
}
with Section("Get graph"):
graph = largest_component(pickle.load(open(PARAM['road_graph'], 'rb')))
nx.set_edge_attributes(graph,
name="met",
values=dict(pd.read_pickle(PARAM['edges_met'])))
with Section("Get trips"):
sql = dict(
table_name="yellow_tripdata_2016-05",
where=
"('2016-05-02 08:00' <= pickup_datetime) and (dropoff_datetime <= '2016-05-02 08:30')",
limit=200,
)
trips = a_effective_metric_manhattan.get_taxidata_trips(**sql)
trips = trips.join(a_effective_metric_manhattan.project(trips, graph),
def experiment(graph_size=32, ntrips=1000, noise=0.2, num_rounds=64):
graph = odd_king_graph(xn=graph_size, yn=graph_size, scale=50)
logger.debug(F"Constructed 'odd king' graph with {graph.number_of_nodes()} nodes")
# nodes = pd.DataFrame(data=nx.get_node_attributes(graph, name="loc"), index=["lat", "lon"]).T
random_state = np.random.RandomState(1)
secret_met = {e: (v * (1 + noise * random_state.random())) for (e, v) in nx.get_edge_attributes(graph, name="len").items()}
nx.set_edge_attributes(graph, name="met", values=secret_met)
assert(sorted(list(graph.nodes)) == sorted(range(graph.number_of_nodes()))), "Expect node labels to be 0, 1, ..."
random_state = np.random.RandomState(2)
trips = pd.DataFrame(
data=((random_state.choice(graph.number_of_nodes(), size=2, replace=False)) for __ in range(ntrips)),
columns=["u", "v"],
)
# logger.warning("Invoking trips.drop_duplicates")
# trips = trips.drop_duplicates()
with Section(F"Collecting {len(trips)} secret trips", out=logger.debug):
with GraphPathDist(graph, edge_weight="met") as pathdist:
# Estimated trajectories of trips
trips = trips.join(
pd.DataFrame(
data=parallel_map(pathdist, progressbar(list(zip(trips.u, trips.v)))),
index=trips.index,
columns=["secret_path", "distance"],
)
)
coverage = pd.Series(dict(Counter(e for path in trips['secret_path'] for e in pairwise(path))))
logger.debug([
F"{nedges} edges x{cov}"
for (cov, nedges) in sorted(Counter(coverage).items(), key=first)
])
# nx.draw(graph, pos=nx.get_node_attributes(graph, name="pos"))
# for (__, trip) in trips.iterrows():
# path = nx.shortest_path(graph, source=trip.u, target=trip.v, weight="met")
# plt.plot(nodes.lon[path], nodes.lat[path], 'b-')
# plt.show()
# Initial metric guess is given by "len"
history = pd.DataFrame({'secret': secret_met, 0: pd.Series(nx.get_edge_attributes(graph, name="len"))})
def cb(info):
if (info.round == (2 ** round(log2(info.round)))):
history[info.round] = info.edges_met
opt = options_refine_effective_metric()
opt.min_trip_distance_m = 0.1
opt.max_trip_distance_m = 1e8
opt.num_rounds = num_rounds
refine_effective_metric(graph, trips, callback=cb, opt=opt)
return history
def refine_effective_metric(
graph: nx.DiGraph,
trips: pd.DataFrame,
opt=options_refine_effective_metric(),
callback=None,
edges_met=None,
skip_rounds=0,
) -> pd.Series:
"""
Returns a pandas series edges_met such that edges_met[E] is the effective length of edge E.
If edges_met is provided it is used as the initial guess (but not modified).
Invalidates the edge attribute opt.temp_graph_metric_attr_name in the graph if present.
"""
if nx.get_edge_attributes(graph, name=opt.temp_graph_metric_attr_name):
logger.warning(
F"Graph edge attributes '{opt.temp_graph_metric_attr_name}' will be invalidates"
)
# Only nontrivial trips that are not too short or too long
trips = trips[trips['u'] != trips['v']]
trips = trips[trips['distance'] >= opt.min_trip_distance_m]
trips = trips[trips['distance'] <= opt.max_trip_distance_m]
logger.debug(F"Trip pool has {len(trips)} trips")
assert ((edges_met is not None) == bool(skip_rounds)
), "Both or none of (edges_met, skip_rounds) should be provided"
# Geographic metric as initial guess / prior
edges_len = pd.Series(data=nx.get_edge_attributes(graph, name="len"),
name="len")
# Effective metric, to be modified
if edges_met is not None:
edges_met = pd.Series(name="met", copy=True, data=edges_met)
skip_rounds = skip_rounds
else:
edges_met = pd.Series(name="met",
copy=True,
data=nx.get_edge_attributes(graph, name="len"))
skip_rounds = 0
for r in range[1 + skip_rounds, opt.num_rounds]:
logger.debug(F"Round {r}")
if any(~edges_met.notna()):
logger.warning(F"There are edges with 'n/a' metric")
with Section("Computing trajectories", out=logger.debug):
nx.set_edge_attributes(graph,
name=opt.temp_graph_metric_attr_name,
values=dict(edges_met))
with GraphPathDist(
graph, edge_weight=opt.temp_graph_metric_attr_name) as gpd:
# Estimated trajectories of trips
traj = pd.DataFrame(
data=parallel_map(gpd,
progressbar(list(zip(trips.u,
trips.v)))),
index=trips.index,
columns=["path", "dist"],
)
# Per-trajectory correction factor
traj['f'] = trips['distance'] / traj['dist']
# # Accept trips/trajectories that are feasibly related
# traj = traj[(0.8 < traj.f) & (traj.f < 1.2)]
logger.debug(
F"Weight correction using {sum(traj.f < 1)}(down) + {sum(traj.f > 1)}(up) trips"
)
with Section("Computing correction factors", out=logger.debug):
with Section("Edges of trajectories"):
edges_loci = dict(zip(edges_met.index, range(len(edges_met))))
edges_of_traj = list(
tuple(edges_loci[e] for e in pairwise(path))
for path in progressbar(traj.path))
with Section("Incidence matrix [trips x edges]"):
M = dok_matrix((len(traj), len(edges_met)), dtype=float)
for (t, edges, f) in zip(range(M.shape[0]), edges_of_traj,
traj.f):
M[t, edges] = f
del edges_of_traj
with Section("Subsample trips"):
I = pd.Series(range(M.shape[0])).sample(
frac=0.5, random_state=opt.random_state)
M = csr_matrix(M)[I, :]
with Section("Compute correction"):
M = csc_matrix(M)
correction = pd.Series(
index=edges_met.index,
data=[(lambda L: (2**np.mean(np.log2(L
if len(L) else 1))))(
M.getcol(j).data)
for j in range(M.shape[1])]).fillna(1)
# Clip and moderate the correction factors
correction = 2**(opt.correction_factor_moderation *
np.log2(correction).clip(lower=-1, upper=+1))
with Section("Applying correction factors", out=logger.debug):
edges_met = edges_met * correction
# Clip extremes, slow-revert to the prior
edges_met = edges_met.clip(lower=(edges_len / 2),
upper=(edges_len * 4))
edges_met = edges_met * (2
**(0.01 * np.log2(edges_len / edges_met)))
if callback:
# # The edges of estimated trajectories
# df = pd.DataFrame.sparse.from_spmatrix(
# data=M,
# index=pd.Series(traj.index, name="Estimated trajectory"),
# columns=pd.Index(edges_met.index, name="Edges")
# ).astype(pd.SparseDtype('float', np.nan))
callback(
SimpleNamespace(graph=graph,
trips=trips,
edges_met=edges_met,
traj=traj,
round=r,
correction=correction))
# Record the estimated metric
nx.set_edge_attributes(graph,
name=opt.temp_graph_metric_attr_name,
values=dict(edges_met))
logger.debug(F"Iteration done")
return edges_met