def speed_on_the_ground(graph: nx.DiGraph, filename: str):
mpl.use("Agg")
with commons.Section("Getting the background OSM map"):
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, name="pos"),
index=["lon", "lat"]).T
extent = maps.ax4(nodes.lat, nodes.lon)
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent))
with mpl.rc_context(PARAM['mpl_style']), commons.Axes() as ax1:
velocity = pd.Series(nx.get_edge_attributes(
graph, name="len")) / pd.Series(
nx.get_edge_attributes(graph, name=PARAM['edge_time_attr']))
cmap = LinearSegmentedColormap.from_list(
name="noname", colors=["brown", "r", "orange", "g"])
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
nx.draw(graph,
ax=ax1,
pos=nx.get_node_attributes(graph, name="pos"),
edgelist=list(velocity.index),
edge_color=list(velocity),
edge_cmap=cmap,
edge_vmin=0,
edge_vmax=7,
with_labels=False,
arrows=False,
node_size=0,
alpha=0.8,
width=0.3)
fn = PARAM['out_images_path'] / commons.myname() / F"{filename}.png"
ax1.figure.savefig(commons.makedirs(fn))
def manhattan_metric_callback(info: SimpleNamespace):
assert (info.round is not None)
assert (info.trips is not None)
assert (info.edges_met is not None)
aboutfile_write({**aboutfile_read(), 'valid': False})
with open(output_edges_fn, 'wb') as fd:
pickle.dump(info.edges_met, fd)
aboutfile_write({
**aboutfile_read(), 'valid':
True,
'days':
list(map(str, days)),
'weekday':
weekday,
'hour':
hour,
'#trips':
len(info.trips),
'round':
info.round,
'timestamp':
datetime.now(tz=timezone.utc).isoformat()
})
fn = makedirs(os.path.join(output_path, F"UV/{info.round:04}.jpg"))
if os.path.isfile(fn):
os.remove(fn)
with nx_draw_met_by_len(info.graph, info.edges_met) as (fig, ax1):
fig.savefig(fn, **{**PARAM['savefig_args'], 'dpi': 180})
def compare_multiple_trajectories(table_name):
mpl.use("Agg")
# Number of trips to plot
N = 10
# Number of trajectories per trip
M = 12
graph = get_road_graph()
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, "loc"), index=["lat", "lon"]).T
edges_len = nx.get_edge_attributes(graph, name="len")
where = "('2016-05-02 08:00' <= pickup_datetime) and (pickup_datetime <= '2016-05-02 09:00')"
trips = get_trip_data(table_name, graph, order="", where=where)
trips = trips.sample(min(N, len(trips)), random_state=1)
logger.debug(F"{len(trips)} trips")
with Section("Getting the background OSM map", out=logger.debug):
extent = maps.ax4(nodes.lat, nodes.lon)
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent))
with plt.style.context({**PARAM['mpl_style'], 'font.size': 5}), Axes() as ax1:
# The background map
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
ax1.axis("off")
ax1.set_xlim(extent[0:2])
ax1.set_ylim(extent[2:4])
for (__, trip) in trips.iterrows():
with Section("Computing candidate trajectories", out=logger.debug):
trajectories = pd.DataFrame(data={'path': [
path
for (__, path) in
zip(range(M), nx.shortest_simple_paths(graph, source=trip.u, target=trip.v))
]})
trajectories['dist'] = [sum(edges_len[e] for e in pairwise(path)) for path in trajectories.path]
trajectories = trajectories.sort_values(by='dist', ascending=False)
marker = dict(markersize=2, markeredgewidth=0.2, markerfacecolor="None")
ax1.plot(trip['pickup_longitude'], trip['pickup_latitude'], 'og', **marker)
ax1.plot(trip['dropoff_longitude'], trip['dropoff_latitude'], 'xr', **marker)
cmap = LinearSegmentedColormap.from_list(name="noname", colors=["g", "orange", "r", "brown"])
colors = cmap(pd.Series(trajectories['dist'] / trip['distance']).rank(pct=True))
for (c, path) in zip(colors, trajectories.path):
(y, x) = nodes.loc[list(path)].values.T
ax1.plot(x, y, c=c, alpha=0.5, lw=0.3)
# Save to file
fn = os.path.join(PARAM['out_images_path'], F"{myname()}/{table_name}.png")
ax1.figure.savefig(makedirs(fn))
def run_experiments() -> pd.DataFrame:
aliquot = datetime.utcnow().strftime("%Y%m%d-%H%M%S")
setups = pd.DataFrame(
# data=list(product([32, 64], [0.1, 0.2, 0.4, 0.8], [(2 ** n) for n in range[7, 13]])),
data=list(product([32, 64], [0.1, 0.2, 0.4, 0.8], [(2 ** n) for n in range[7, 10]])),
# data=list(product([64], [0.5], [2 ** 13])),
# data=list(product([(2 ** n) for n in range[2, 6]], [0.1, 0.2, 0.4], [100, 1000, 10000])),
# data=list(product([4, 8, 16], [0.1], [10, 100])),
columns=["graph_size", "noise", "ntrips"],
)
logger.debug('\n'.join(map(str, ["Experiments:", setups])))
for setup in setups.itertuples(index=False):
# Preserve datatypes
setup = dict(setup._asdict())
# Alternative:
# setup = setup.astype({'graph_size': int, 'noise': float, 'ntrips': int})
with Section(F"Experiment {setup} is on", out=logger.info):
# https://www.dataquest.io/blog/settingwithcopywarning/
with pd.option_context('mode.chained_assignment', None):
history = experiment(**setup, num_rounds=64)
with open(makedirs(PARAM['out_experiment_results'].format(aliquot=aliquot, ext="pkl")), 'ab') as fd:
pickle.dump({**setup, 'history': history}, fd)
# results.to_json(makedirs(PARAM['out_experiment_results'].format(aliquot=aliquot, ext="json")))
with open(makedirs(PARAM['out_experiment_results'].format(aliquot=aliquot, ext="json")), 'w') as fd:
json.dump(
{
'setups': setups.to_json(),
'script': this_module_body(),
'timestamp': datetime.now(tz=timezone.utc).isoformat(),
},
fd
)
return setups
def trip_distance_vs_shortest(table_name):
mpl.use("Agg")
graph = get_road_graph()
trips = get_trip_data(table_name, graph)
with Section("Computing shortest distances", out=logger.debug):
trips = trips.join(
pd.DataFrame(
data=parallel_map(GraphPathDist(graph, edge_weight="len"), zip(trips.u, trips.v)),
columns=['path', 'shortest'], index=trips.index,
)
)
# On-graph distance vs reported distance [meters]
df: pd.DataFrame
df = pd.DataFrame(data=dict(
reported=(trips['distance']),
shortest=(trips['shortest']),
))
# Convert to [km] and stay below 10km
df = df.applymap(lambda x: (x / 1e3))
df = df.applymap(lambda km: (km if (km < 10) else np.nan)).dropna()
# Hour of the day
df['h'] = trips['pickup_datetime'].dt.hour
with plt.style.context(PARAM['mpl_style']):
with Axes() as ax1:
ax1.set_aspect(aspect="equal", adjustable="box")
ax1.grid()
ax1.plot(*(2 * [[0, df[['reported', 'shortest']].values.max()]]), c='k', ls='--', lw=0.5, zorder=100)
for (h, hdf) in df.groupby(df['h']):
c = plt.get_cmap("twilight_shifted")([h / 24])
ax1.scatter(
hdf['reported'], hdf['shortest'],
c=c, s=3, alpha=0.8, lw=0, zorder=10,
label=(F"{len(hdf)} trips at {h}h")
)
ax1.set_xlabel("Reported distance, km")
ax1.set_ylabel("Naive graph distance, km")
ax1.set_xticks(range(11))
ax1.set_yticks(range(11))
ax1.legend()
# Save to file
fn = os.path.join(PARAM['out_images_path'], F"{myname()}/{table_name}.png")
ax1.figure.savefig(makedirs(fn))
# Meta info
json.dump({'number_of_datapoints': len(df)}, open((fn + ".txt"), 'w'))
def trip_trajectories_ingraph(table_name):
mpl.use("Agg")
# Max number of trajectories to plot
N = 1000
graph = get_road_graph()
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, "loc"), index=["lat", "lon"]).T
trips = get_trip_data(table_name, graph)
trips = trips.sample(min(N, len(trips)))
logger.debug(F"{len(trips)} trips")
logger.debug("Computing trajectories")
trajectories = parallel_map(GraphPathDist(graph).path_only, zip(trips.u, trips.v))
with Section("Getting the background OSM map", out=logger.debug):
extent = maps.ax4(nodes.lat, nodes.lon)
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent))
with plt.style.context({**PARAM['mpl_style'], 'font.size': 5}):
with Axes() as ax1:
# The background map
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
ax1.axis("off")
ax1.set_xlim(extent[0:2])
ax1.set_ylim(extent[2:4])
c = 'b'
if ("green" in table_name): c = "green"
if ("yello" in table_name): c = "orange"
logger.debug("Plotting trajectories")
for traj in trajectories:
(y, x) = nodes.loc[list(traj)].values.T
ax1.plot(x, y, c=c, alpha=0.1, lw=0.3)
# Save to file
fn = os.path.join(PARAM['out_images_path'], F"{myname()}/{table_name}.png")
ax1.figure.savefig(makedirs(fn))
# Meta info
json.dump({'number_of_trajectories': len(trips)}, open((fn + ".txt"), 'w'))
def trip_trajectories_velocity(table_name):
mpl.use("Agg")
# Max number of trajectories to use
N = 10000
graph = get_road_graph()
nodes = pd.DataFrame(data=nx.get_node_attributes(graph, "loc"), index=["lat", "lon"]).T
edge_name = pd.Series(nx.get_edge_attributes(graph, name="name"))
where = "('2016-05-02 08:00' <= pickup_datetime) and (pickup_datetime <= '2016-05-02 09:00')"
trips = get_trip_data(table_name, graph, order="", limit=N, where=where)
trips['velocity'] = trips['distance'] / trips['duration/s']
trips = trips.sort_values(by='velocity', ascending=True)
logger.debug(F"{len(trips)} trips")
with Section("Computing estimated trajectories", out=logger.debug):
trips['traj'] = parallel_map(GraphPathDist(graph).path_only, zip(trips.u, trips.v))
with Section("Getting the background OSM map", out=logger.debug):
extent = maps.ax4(nodes.lat, nodes.lon)
osmap = maps.get_map_by_bbox(maps.ax2mb(*extent))
with Section("Computing edge velocities", out=logger.debug):
edge_vel = defaultdict(list)
for (traj, v) in zip(trips.traj, trips.velocity):
for e in pairwise(traj):
edge_vel[e].append(v)
edge_vel = pd.Series({e: np.mean(v or np.nan) for (e, v) in edge_vel.items()}, index=graph.edges)
edge_vel = edge_vel.dropna()
with plt.style.context({**PARAM['mpl_style'], 'font.size': 5}), Axes() as ax1:
# The background map
ax1.imshow(osmap, extent=extent, interpolation='quadric', zorder=-100)
ax1.axis("off")
ax1.set_xlim(extent[0:2])
ax1.set_ylim(extent[2:4])
cmap_velocity = LinearSegmentedColormap.from_list(name="noname", colors=["brown", "r", "orange", "g"])
# marker = dict(markersize=0.5, markeredgewidth=0.1, markerfacecolor="None")
# ax1.plot(trips['pickup_longitude'], trips['pickup_latitude'], 'og', **marker)
# ax1.plot(trips['dropoff_longitude'], trips['dropoff_latitude'], 'xr', **marker)
# for e in edge_name[edge_name == "65th Street Transverse"].index:
# print(e, edge_vel[e])
edge_vel: pd.Series
# edge_vel = edge_vel.rank(pct=True)
edge_vel = edge_vel.clip(lower=2, upper=6).round()
edge_vel = (edge_vel - edge_vel.min()) / (edge_vel.max() - edge_vel.min())
edge_vel = edge_vel.apply(cmap_velocity)
nx.draw_networkx_edges(
graph.edge_subgraph(edge_vel.index),
ax=ax1,
pos=nx.get_node_attributes(graph, name="pos"),
edge_list=list(edge_vel.index),
edge_color=list(edge_vel),
# edge_cmap=cmap_velocity,
# vmin=0, vmax=1,
with_labels=False, arrows=False, node_size=0, alpha=0.8, width=0.3,
)
# Save to file
fn = os.path.join(PARAM['out_images_path'], F"{myname()}/{table_name}.png")
ax1.figure.savefig(makedirs(fn))
# Meta info
json.dump({'number_of_trajectories': len(trips)}, open((fn + ".txt"), 'w'))
logger.basicConfig(
level=logger.DEBUG,
format="%(levelname)-8s [%(asctime)s] : %(message)s",
datefmt="%Y%m%d %H:%M:%S %Z",
)
logger.getLogger('matplotlib').setLevel(logger.WARNING)
logger.getLogger('PIL').setLevel(logger.WARNING)
# ~~~~ SETTINGS ~~~~ #
PARAM = {
'taxidata': "data/taxidata/sqlite/UV/db.db",
'road_graph': "data/road_graph/UV/nx_digraph_naive.pkl",
'out_images_path': makedirs("exploration/"),
'mpl_style': {
'font.size': 3,
'xtick.major.size': 2,
'ytick.major.size': 0,
'xtick.major.pad': 1,
'ytick.major.pad': 1,
'savefig.bbox': "tight",
'savefig.pad_inches': 0,
'savefig.dpi': 300,
},
}
IFILE = {
#'mapmatched' : "OUTPUT/14/mapmatched/{scenario}/{routeid}-{direction}/UV/{mapmatch_uuid}.{ext}",
'segment_by_route': "OUTPUT/13/{scenario}/byroute/UV/{routeid}-{dir}.json",
'MOTC_routes':
"OUTPUT/00/ORIGINAL_MOTC/{City}/CityBusApi_StopOfRoute.json",
}
## =================== OUTPUT :
OFILE = {
'timetable_json':
"OUTPUT/17/timetable/{scenario}/json/{routeid}-{dir}.json",
}
commons.makedirs(OFILE)
## ==================== PARAM :
PARAM = {
#'mapbox_api_token' : open(".credentials/UV/mapbox-token.txt", 'r').read(),
# When is the bus run too short at the tails?
'tail_eta_patch_dist': 50, # meters
# Have at least those many waypoints close to the run
'min_near_run': 3,
'n_parallel_jobs': min(12, ceil(cpu_count() / 1.5)),
}
## ====================== AUX :
def plot_results():
mpl.use("Agg")
def summary(history: pd.DataFrame) -> pd.DataFrame:
history = history.div(history['secret'], axis=0).drop(columns=['secret', 0])
history = (100 * history.transform(np.log10).mean(axis=0)).transform(np.abs)
return history
cat = pd.DataFrame(glob(PARAM['out_experiment_results'].format(aliquot="*", ext="*")), columns=["file"])
cat = cat.assign(dir=list(map(os.path.dirname, cat.file)))
cat = cat.assign(sig=list(map(os.path.basename, cat.dir)))
cat = cat.assign(ext=list(map(last, map(os.path.splitext, cat.file))))
cat = cat.pivot(index='dir', columns='ext', values='file')
for (folder, meta) in cat.iterrows():
logger.info(F"Folder: {os.path.relpath(folder, os.path.dirname(__file__))}")
df: pd.DataFrame
df = pd.DataFrame()
with open(meta['.pkl'], 'rb') as fd:
with suppress(EOFError):
while df is not None:
df = df.append(pickle.load(fd), ignore_index=True)
df = df[df.ntrips > 10]
df = df[~df.history.isna()]
df = df.astype({'noise': float, 'graph_size': int, 'ntrips': int})
for (graph_size, df1) in df.groupby(df.graph_size):
for (noise, df2) in df1.groupby(df1.noise):
image_file = makedirs(os.path.join(folder, "images", F"graph_size={graph_size}", F"noise={noise}".replace(".", "p"), "round.{ext}"))
fig: plt.Figure
ax1: plt.Axes
(fig, ax1) = plt.subplots()
for (ntrips, history) in zip(df2['ntrips'], map(summary, df2['history'])):
ax1.plot(history, marker='.', ls='--', label=ntrips)
ax1.set_xscale("log")
ax1.set_yscale("log")
ax1.set_ylabel("Geometric average relative error, %")
ax1.set_ylim(1e-1, 1e1)
ax1.grid()
ax1.set_title(F"Graph size: {graph_size}, noise: {noise}")
ax1.legend()
fig.savefig(image_file.format(ext="png"), **PARAM['savefig_args'])
plt.close(fig)
for (graph_size, df1) in df.groupby(df.graph_size):
image_file = makedirs(os.path.join(folder, "images", F"graph_size={graph_size}", "ntrips.{ext}"))
fig: plt.Figure
ax1: plt.Axes
(fig, ax1) = plt.subplots()
for (noise, df2) in df1.groupby(df1.noise):
history = pd.Series(index=df2['ntrips'], data=list(map(min, map(summary, df2['history']))))
ax1.plot(history, marker='.', ls='--', label=noise)
ax1.set_xlabel("Number of trips")
ax1.set_ylabel("Geometric average of relative error, %")
ax1.set_xscale("log")
ax1.set_yscale("log")
ax1.set_ylim(1e-1, 1e1)
ax1.grid()
ax1.set_title(F"Graph size: {graph_size}")
ax1.legend()
fig.savefig(image_file.format(ext="png"), **PARAM['savefig_args'])
plt.close(fig)
from types import SimpleNamespace
from collections import Counter
from itertools import product
from more_itertools import pairwise, first, last
from contextlib import suppress
from progressbar import progressbar
import seaborn as sb
import matplotlib as mpl
import matplotlib.pyplot as plt
from percache import Cache
cache = Cache(makedirs(os.path.join(os.path.dirname(__file__), "synthetic/UV/percache_runs")), livesync=True)
# ~~~~ LOGGING ~~~~ #
import logging as logger
logger.basicConfig(level=logger.DEBUG, format="%(levelname)-8s [%(asctime)s] @%(funcName)s : %(message)s", datefmt="%Y%m%d %H:%M:%S %Z")
logger.getLogger('matplotlib').setLevel(logger.WARNING)
logger.getLogger('PIL').setLevel(logger.WARNING)
# ~~~~ SETTINGS ~~~~ #
PARAM = {
'out_experiment_results': os.path.join(os.path.dirname(__file__), "synthetic", "{aliquot}", "results.{ext}"),
'savefig_args': dict(bbox_inches='tight', pad_inches=0, jpeg_quality=0.9, dpi=300),
}
def mapmatch_runs(scenario, runs):
# Road network (main graph component) with nearest-neighbor tree for the nodes
g: nx.DiGraph
(g, knn) = commons.inspect(['g', 'knn'])(pickle.load(
open(IFILE['OSM_graph_file'], 'rb'))['main_component_with_knn'])
g = trim_graph_to_busable(g)
# Nearest edges computer
kne = (lambda q: graph.estimate_kne(g, knn, q, ke=20))
mpl.use('Agg')
mpl.rcParams['figure.max_open_warning'] = 100
import matplotlib.pyplot as plt
#
def make_figure(result) -> dict:
ax: plt.Axes
if ('plt' in result):
(fig, ax) = commons.inspect({'plt': ('fig', 'ax')})(result)
else:
(fig, ax) = plt.subplots()
result['plt'] = {'fig': fig, 'ax': ax}
result['auto_close_fig'] = commons.UponDel(lambda: plt.close(fig))
ax.cla()
ax.set_title("{} ({}%)".format(
result['status'], math.floor(100 * result.get('progress', 0))))
if ('waypoints_all' in result):
(y, x) = zip(*result['waypoints_all'])
ax.plot(x, y, 'o', c='m', markersize=2)
if ('geo_path' in result):
(y, x) = zip(*result['geo_path'])
ax.plot(x, y, 'b--', linewidth=2, zorder=100)
return result['plt']
#
def mm_callback(result) -> None:
if (result['status'] == "opti"):
if (dt.datetime.now() < result.get('nfu', dt.datetime.min)):
return
# Log into a GPX file
if ('waypoints_all' in result):
with open(OFILE['progress'].format(ext="gpx"), 'w') as fd:
fd.write(
graph.simple_gpx(result['waypoints_all'],
[result.get('geo_path', [])]).to_xml())
# Save figure
make_figure(result)
with open(OFILE['progress'].format(ext="png"), 'wb') as fd:
fig.savefig(fd, bbox_inches='tight', pad_inches=0)
# Next figure update
result['nfu'] = dt.datetime.now() + dt.timedelta(seconds=5)
# Collect all bus runs
runs_by_runid = {run[KEYS.runid]: run for run in runs}
# Collect all waypoints
waypoints_by_runid = {
runid: list(map(tuple, run[KEYS.pos]))
for (runid, run) in runs_by_runid.items()
}
#commons.logger.debug(json.dumps(runs, indent=2))
commons.logger.info("Running mapmatch on {} runs".format(len(runs)))
# MAPMATCH RUNS
results = graph.mapmatch(waypoints_by_runid,
g,
kne,
knn=knn,
callback=None,
stubborn=0.2,
many_partial=True)
for result in results:
commons.logger.info("Got mapmatch with waypoints {}".format(
result['waypoints_used']))
# The run on which mapmatch operated
run = runs_by_runid[result['waypoint_setid']]
# Collect initial info about the mapmatch attempt
mapmatch_attempt = {
k: run[k]
for k in [KEYS.routeid, KEYS.dir, KEYS.runid, KEYS.busid]
}
# Attach a unique identifier for this mapmatch
mapmatch_attempt['MapMatchUUID'] = uuid.uuid4().hex
# Filename without the extension
fn = OFILE['mapmatched'].format(
scenario=scenario,
routeid=mapmatch_attempt[KEYS.routeid],
direction=mapmatch_attempt[KEYS.dir],
mapmatch_uuid=mapmatch_attempt['MapMatchUUID'],
ext="{ext}")
# Copy relevant fields from the mapmatcher result
for k in ['waypoints_used', 'path', 'geo_path', 'mapmatcher_version']:
mapmatch_attempt[k] = result[k]
# Save the result in different formats, in this directory
commons.makedirs(fn.format(ext='~~~'))
# o) Image
try:
# Make and save the figure
fig = make_figure(result)['fig']
with open(fn.format(ext="png"), 'wb') as fd:
fig.savefig(fd, bbox_inches='tight', pad_inches=0)
except Exception as e:
commons.logger.warning("Could not save figure {} ({})".format(
fn.format(ext="png"), e))
# o) JSON
try:
with open(fn.format(ext="json"), 'w') as fd:
json.dump(mapmatch_attempt, fd)
except Exception as e:
commons.logger.warning(
"Failed to write mapmatch file {} ({})".format(
fn.format(ext="json"), e))
# o) GPX
try:
with open(fn.format(ext="gpx"), 'w') as fd:
fd.write(
graph.simple_gpx(mapmatch_attempt['waypoints_used'],
[mapmatch_attempt['geo_path']]).to_xml())
except Exception as e:
commons.logger.warning("Failed to write GPX file {} ({})".format(
fn.format(ext="gpx"), e))
time.sleep(1)
def compute_metric_for_table(table_name):
# Get the database table timespan
with sqlite3.connect(PARAM['taxidata']) as con:
sql = F"SELECT min(pickup_datetime), max(dropoff_datetime) FROM [{table_name}]"
dates = pd.date_range(*pd.read_sql_query(sql=sql, con=con).iloc[0])
#
# for ((weekday, days), hour) in product(dates.groupby(dates.weekday).items(), [8]):
for ((weekday, days),
hour) in product(dates.groupby(dates.weekday).items(), range[0, 23]):
logger.debug(F"weekday/hour = {weekday}/{hour} over {len(days)} days")
# Filenames for output
output_path = makedirs(
os.path.join(PARAM['out_metric'],
F"{table_name}/{weekday}/{hour:02}/"))
output_edges_fn = os.path.join(output_path, "edges_met.pkl")
output_about_fn = os.path.join(output_path, "edges_met.json")
def aboutfile_read():
try:
with open(output_about_fn, 'r') as fd:
return json.load(fd)
except FileNotFoundError:
return {'locked': False}
def aboutfile_write(info):
with open(output_about_fn, 'w') as fd:
json.dump(info, fd)
def manhattan_metric_callback(info: SimpleNamespace):
assert (info.round is not None)
assert (info.trips is not None)
assert (info.edges_met is not None)
aboutfile_write({**aboutfile_read(), 'valid': False})
with open(output_edges_fn, 'wb') as fd:
pickle.dump(info.edges_met, fd)
aboutfile_write({
**aboutfile_read(), 'valid':
True,
'days':
list(map(str, days)),
'weekday':
weekday,
'hour':
hour,
'#trips':
len(info.trips),
'round':
info.round,
'timestamp':
datetime.now(tz=timezone.utc).isoformat()
})
fn = makedirs(os.path.join(output_path, F"UV/{info.round:04}.jpg"))
if os.path.isfile(fn):
os.remove(fn)
with nx_draw_met_by_len(info.graph, info.edges_met) as (fig, ax1):
fig.savefig(fn, **{**PARAM['savefig_args'], 'dpi': 180})
# Defaults
skip_rounds = 0
edges_met = None
# Job status
about = aboutfile_read()
if about.get('locked', True):
# Somebody may be working on this now
logger.info(F"Locked(?) {output_about_fn} exists -- skipping")
continue
if (about.get('valid') == False):
logger.warning(
F"Job {output_about_fn} explicitly flagged as invalid -- skipping"
)
continue
if about.get('valid'):
logger.info(F"Trying to resume {output_about_fn}")
skip_rounds = about['round']
with open(output_edges_fn, 'rb') as fd:
edges_met = pickle.load(fd)
try:
# Quickly, reserve the job
logger.debug("Marking job as locked")
aboutfile_write({**about, 'locked': True, 'exception': None})
# Load and sanitize the road graph
graph = get_road_graph()
# Taxidata query details
where = " OR ".join(
"(('{a}' <= pickup_datetime) AND (dropoff_datetime < '{b}'))".
format(
a=(day + pd.Timedelta(hour + 0, unit='h')),
b=(day + pd.Timedelta(hour + 1, unit='h')),
) for day in days)
# Query the taxidata database
trips = get_taxidata_trips(table_name, where=where)
# Attach nearest nodes
trips = trips.join(project(trips, graph), how="inner")
#
opt = options_refine_effective_metric()
opt.min_trip_distance_m = 200
opt.max_trip_distance_m = 7e3
opt.num_rounds = 100
# Run the metric computation loop
# Note: The callback function records the results
refine_effective_metric(graph,
trips,
opt=opt,
callback=manhattan_metric_callback,
skip_rounds=skip_rounds,
edges_met=edges_met)
except:
(exc_type, value, traceback) = sys.exc_info()
aboutfile_write({
**aboutfile_read(), 'exception': (exc_type.__name__)
})
raise
finally:
logger.debug("Marking job as non-locked")
aboutfile_write({**aboutfile_read(), 'locked': False})
fn)
# ~~~~ SETTINGS ~~~~ #
PARAM = {
'taxidata':
datapath("taxidata/sqlite/UV/db.db"),
'road_graph':
datapath("road_graph/UV/nx_digraph_naive.pkl"),
# Tolerance for accepting a nearest node (distance in meters)
'tol_nearest_nodes/m':
20,
'out_metric':
makedirs(os.path.join(os.path.dirname(__file__), "manhattan_metric/")),
'savefig_args':
dict(bbox_inches='tight', pad_inches=0, jpeg_quality=0.9, dpi=300),
}
# ~~~~ HELPERS ~~~~ #
# ~~~~ GRAPHICS ~~~~ #
@contextmanager
@retry(KeyboardInterrupt, tries=2, delay=1)
def nx_draw_met_by_len(graph, edges_met=None, mpl_backend="Agg", printer=None):
mpl.use(mpl_backend)
with Section("Preparing to draw graph", out=printer):
