def calculate_additional_columns_for_journey(self):
if not self.jdm:
self.jdm = JourneyDataManager(self.gtfs_dir, journey_db_path=self.journey_dir,
routing_params=self.routing_params, track_vehicle_legs=TRACK_VEHICLE_LEGS,
track_route=TRACK_ROUTE)
self.jdm.populate_additional_journey_columns()
self.jdm.compute_and_store_travel_impedance_measures(self.analysis_start_time, self.analysis_end_time, TRAVEL_IMPEDANCE_STORE_PATH)
class TestJourneyData(TestCase):
# noinspection PyAttributeOutsideInit
def _import_sample_gtfs_db(self):
import_gtfs([
os.path.join(os.path.dirname(__file__),
"../../test/test_data/test_gtfs.zip")
], self.gtfs_path)
def _remove_routing_test_data_directory_if_exists(self):
try:
shutil.rmtree(self.routing_tmp_test_data_dir)
except FileNotFoundError:
pass
def _create_routing_test_data_directory(self):
if not os.path.exists(self.routing_tmp_test_data_dir):
os.makedirs(self.routing_tmp_test_data_dir)
def setUp(self):
self.routing_tmp_test_data_dir = "./tmp_routing_test_data/"
self.gtfs_path = os.path.join(self.routing_tmp_test_data_dir,
"test_gtfs.sqlite")
self.data_store_path = os.path.join(self.routing_tmp_test_data_dir,
"test_data_store.sqlite")
self._remove_routing_test_data_directory_if_exists()
self._create_routing_test_data_directory()
self._import_sample_gtfs_db()
self.jdm = JourneyDataManager(
self.gtfs_path,
os.path.join(self.routing_tmp_test_data_dir,
"test_journeys.sqlite"),
routing_params={"track_vehicle_legs": True})
def tearDown(self):
self._remove_routing_test_data_directory_if_exists()
def test_boardings_computations_based_on_journeys(self):
# input some journeys
destination_stop = 1
origin_stop = 2
self.jdm.import_journey_data_for_target_stop(
destination_stop, {
origin_stop: [
LabelTimeWithBoardingsCount(1, 2, 1, True),
LabelTimeWithBoardingsCount(2, 3, 2, True)
]
},
enforce_synchronous_writes=True)
self.jdm.compute_and_store_travel_impedance_measures(
0, 2, self.data_store_path)
store = TravelImpedanceDataStore(self.data_store_path)
df = store.read_data_as_dataframe("temporal_distance")
self.assertAlmostEqual(df.iloc[0]["min"], 1)
self.assertAlmostEqual(df.iloc[0]["mean"], 1.5)
self.assertAlmostEqual(df.iloc[0]["max"], 2.0)
self.assertIn(df.iloc[0]["median"], [1, 2, 1.0, 1.5, 2.0])
def setUp(self):
self.routing_tmp_test_data_dir = "./tmp_routing_test_data/"
self.gtfs_path = os.path.join(self.routing_tmp_test_data_dir,
"test_gtfs.sqlite")
self._remove_routing_test_data_directory_if_exists()
self._create_routing_test_data_directory()
self._import_sample_gtfs_db()
self.jdm = JourneyDataManager(
self.gtfs_path,
os.path.join(self.routing_tmp_test_data_dir,
"test_journeys.sqlite"),
routing_params={"track_vehicle_legs": True})
def store_pickle_in_db(self):
self.jdm = JourneyDataManager(self.gtfs_dir, journey_db_path=self.journey_dir,
routing_params=self.routing_params, track_vehicle_legs=TRACK_VEHICLE_LEGS,
track_route=TRACK_ROUTE)
for root, dirs, files in os.walk(self.pickle_dir):
for target_file in files:
target = target_file.replace(".pickle", "")
if not target in self.jdm.get_targets_having_journeys():
print("target: ", target)
profiles = pickle.load(open(os.path.join(root, target_file), 'rb'))
self.jdm.import_journey_data_for_target_stop(int(target), profiles)
else:
print("skipping: ", target, " already in db")
self.jdm.create_indices()
def __init__(self, feed_dict, routing_params):
self.pickle = PICKLE
self.gtfs_dir = feed_dict["gtfs_dir"]
self.G = GTFS(feed_dict["gtfs_dir"])
self.tz = self.G.get_timezone_name()
self.journey_dir = feed_dict["journey_dir"]
self.day_start = feed_dict["day_start"]
self.day_end = feed_dict["day_end"]
self.routing_start_time = feed_dict["routing_start_time"]
self.routing_end_time = feed_dict["routing_end_time"]
self.analysis_start_time = feed_dict["analysis_start_time"]
self.analysis_end_time = feed_dict["analysis_end_time"]
self.pickle_dir = feed_dict["pickle_dir"]
self.routing_params = routing_params
self.jdm = None
if not self.pickle:
self.jdm = JourneyDataManager(os.path.join(GTFS_DB_WORK_DIR, GTFS_DB_FNAME),
journey_db_path=os.path.join(RESULTS_DIR, JOURNEY_DB_FNAME),
routing_params=self.routing_params, track_vehicle_legs=TRACK_VEHICLE_LEGS,
track_route=TRACK_ROUTE)
def calculate_comparison_measures(self):
if not self.jdm:
self.jdm = JourneyDataManager(self.gtfs_dir, journey_db_path=self.journey_dir,
routing_params=self.routing_params, track_vehicle_legs=TRACK_VEHICLE_LEGS,
track_route=TRACK_ROUTE)
prev_dict = None
prev_key = None
before_db_tuple = None
after_db_tuple = None
for (key, feed_dict) in FEED_LIST:
if prev_dict:
if feed_dict["feed_seq"] < prev_dict["feed_seq"]:
after_db_tuple = (feed_dict["journey_dir"], key)
before_db_tuple = (prev_dict["journey_dir"], prev_key)
else:
before_db_tuple = (feed_dict["journey_dir"], key)
after_db_tuple = (prev_dict["journey_dir"], prev_key)
prev_dict = feed_dict
prev_key = key
self.jdm.initialize_comparison_tables(DIFF_PATH, before_db_tuple, after_db_tuple)
def script(self):
self.get_profile_data()
journey_analyzer = JourneyDataManager(TARGET_STOPS,
JOURNEY_DATA_DIR,
GTFS_DATA_BASEDIR,
ROUTING_PARAMS,
track_route=True,
close_connection=False)
journey_analyzer.import_journey_data_for_target_stop(self.profiles)
journey_analyzer.create_indices()
if False:
journey_analyzer.add_fastest_path_column()
"""
class AllToAllRoutingPipeline:
def __init__(self, feed_dict, routing_params):
self.pickle = PICKLE
self.gtfs_dir = feed_dict["gtfs_dir"]
self.G = GTFS(feed_dict["gtfs_dir"])
self.tz = self.G.get_timezone_name()
self.journey_dir = feed_dict["journey_dir"]
self.day_start = feed_dict["day_start"]
self.day_end = feed_dict["day_end"]
self.routing_start_time = feed_dict["routing_start_time"]
self.routing_end_time = feed_dict["routing_end_time"]
self.analysis_start_time = feed_dict["analysis_start_time"]
self.analysis_end_time = feed_dict["analysis_end_time"]
self.pickle_dir = feed_dict["pickle_dir"]
self.routing_params = routing_params
self.jdm = None
if not self.pickle:
self.jdm = JourneyDataManager(os.path.join(GTFS_DB_WORK_DIR, GTFS_DB_FNAME),
journey_db_path=os.path.join(RESULTS_DIR, JOURNEY_DB_FNAME),
routing_params=self.routing_params, track_vehicle_legs=TRACK_VEHICLE_LEGS,
track_route=TRACK_ROUTE)
def get_all_events(self):
print("Retrieving transit events")
connections = []
for e in self.G.generate_routable_transit_events(start_time_ut=self.routing_start_time,
end_time_ut=self.routing_end_time):
connections.append(Connection(int(e.from_stop_I),
int(e.to_stop_I),
int(e.dep_time_ut),
int(e.arr_time_ut),
int(e.trip_I),
int(e.seq)))
assert (len(connections) == len(set(connections)))
print("scheduled events:", len(connections))
print("Retrieving walking network")
net = walk_transfer_stop_to_stop_network(self.G, max_link_distance=CUTOFF_DISTANCE)
print("net edges: ", len(net.edges()))
return net, connections
@timeit
def loop_trough_targets_and_run_routing(self, targets, slurm_array_i):
net, connections = self.get_all_events()
csp = None
for target in targets:
print(target)
if csp is None:
csp = MultiObjectivePseudoCSAProfiler(connections, target, walk_network=net,
end_time_ut=self.routing_end_time,
transfer_margin=TRANSFER_MARGIN,
start_time_ut=self.routing_start_time, walk_speed=WALK_SPEED,
verbose=True, track_vehicle_legs=TRACK_VEHICLE_LEGS,
track_time=TRACK_TIME, track_route=TRACK_ROUTE)
else:
csp.reset([target])
csp.run()
profiles = dict(csp.stop_profiles)
if self.pickle:
self._pickle_results(profiles, slurm_array_i, target)
else:
self.jdm.import_journey_data_for_target_stop(target, profiles)
profiles = None
gc.collect()
@timeit
def loop_trough_targets_and_run_routing_with_route(self, targets, slurm_array_i):
net, connections = self.get_all_events()
csp = None
for target in targets:
print("target: ", target)
if csp is None:
csp = MultiObjectivePseudoCSAProfiler(connections, target, walk_network=net,
end_time_ut=self.routing_end_time,
transfer_margin=TRANSFER_MARGIN,
start_time_ut=self.routing_start_time, walk_speed=WALK_SPEED,
verbose=True, track_vehicle_legs=TRACK_VEHICLE_LEGS,
track_time=TRACK_TIME, track_route=TRACK_ROUTE)
else:
csp.reset([target])
csp.run()
profiles = dict(csp.stop_profiles)
if self.pickle:
self._pickle_results(profiles, slurm_array_i, target)
else:
self.jdm.import_journey_data_for_target_stop(target, profiles)
profiles = None
gc.collect()
@timeit
def _pickle_results(self, profiles, pickle_subdir, target):
pickle_path = makedirs(os.path.join(self.pickle_dir, str(pickle_subdir)))
pickle_path = os.path.join(pickle_path, str(target) + ".pickle")
profiles = dict((key, value.get_final_optimal_labels()) for (key, value) in profiles.items())
"""for key, values in profiles.items():
values.sort(key=lambda x: x.departure_time, reverse=True)
new_values = compute_pareto_front(values)
profiles[key] = new_values
"""
pickle.dump(profiles, open(pickle_path, 'wb'), -1)
profiles = None
gc.collect()
def get_list_of_stops(self, where=''):
df = self.G.execute_custom_query_pandas("SELECT stop_I FROM stops " + where + " ORDER BY stop_I")
return df
@timeit
def store_pickle_in_db(self):
self.jdm = JourneyDataManager(self.gtfs_dir, journey_db_path=self.journey_dir,
routing_params=self.routing_params, track_vehicle_legs=TRACK_VEHICLE_LEGS,
track_route=TRACK_ROUTE)
for root, dirs, files in os.walk(self.pickle_dir):
for target_file in files:
target = target_file.replace(".pickle", "")
if not target in self.jdm.get_targets_having_journeys():
print("target: ", target)
profiles = pickle.load(open(os.path.join(root, target_file), 'rb'))
self.jdm.import_journey_data_for_target_stop(int(target), profiles)
else:
print("skipping: ", target, " already in db")
self.jdm.create_indices()
def calculate_additional_columns_for_journey(self):
if not self.jdm:
self.jdm = JourneyDataManager(self.gtfs_dir, journey_db_path=self.journey_dir,
routing_params=self.routing_params, track_vehicle_legs=TRACK_VEHICLE_LEGS,
track_route=TRACK_ROUTE)
self.jdm.populate_additional_journey_columns()
self.jdm.compute_and_store_travel_impedance_measures(self.analysis_start_time, self.analysis_end_time, TRAVEL_IMPEDANCE_STORE_PATH)
def calculate_comparison_measures(self):
if not self.jdm:
self.jdm = JourneyDataManager(self.gtfs_dir, journey_db_path=self.journey_dir,
routing_params=self.routing_params, track_vehicle_legs=TRACK_VEHICLE_LEGS,
track_route=TRACK_ROUTE)
prev_dict = None
prev_key = None
before_db_tuple = None
after_db_tuple = None
for (key, feed_dict) in FEED_LIST:
if prev_dict:
if feed_dict["feed_seq"] < prev_dict["feed_seq"]:
after_db_tuple = (feed_dict["journey_dir"], key)
before_db_tuple = (prev_dict["journey_dir"], prev_key)
else:
before_db_tuple = (feed_dict["journey_dir"], key)
after_db_tuple = (prev_dict["journey_dir"], prev_key)
prev_dict = feed_dict
prev_key = key
self.jdm.initialize_comparison_tables(DIFF_PATH, before_db_tuple, after_db_tuple)
def main(cmd, args):
print(cmd)
if cmd == "run_preparations":
for (feed, feed_dict) in FEED_LIST:
jdm = JourneyDataManager(feed_dict["gtfs_dir"], None, routing_params(feed),
track_vehicle_legs=TRACK_VEHICLE_LEGS, track_route=TRACK_ROUTE)
jdm.initialize_database(feed_dict["journey_dir"])
elif cmd[:5] == "run_w":
feed = args[0]
slurm_array_i = args[1]
slurm_array_length = args[2]
slurm_array_i = int(slurm_array_i)
slurm_array_length = int(slurm_array_length)
assert(slurm_array_i < slurm_array_length)
if cmd == "run_routing":
a2a_pipeline = AllToAllRoutingPipeline()
df = a2a_pipeline.get_list_of_stops()
nodes = df['stop_I'].sample(frac=1, random_state=123).values
parts = split_into_equal_length_parts(nodes, slurm_array_length)
targets = parts[slurm_array_i]
assert not TRACK_ROUTE
a2a_pipeline.loop_trough_targets_and_run_routing(targets, slurm_array_i)
elif cmd == "run_with_routes_slurm":
print(feed)
a2a_pipeline = AllToAllRoutingPipeline(FEED_DICT[feed], routing_params(feed))
df = a2a_pipeline.get_list_of_stops(where="WHERE code = '__target__'")
nodes = df['stop_I'].sample(frac=1, random_state=123).values
parts = split_into_equal_length_parts(nodes, slurm_array_length)
targets = parts[slurm_array_i]
assert TRACK_ROUTE
a2a_pipeline.loop_trough_targets_and_run_routing_with_route(targets, slurm_array_i)
elif cmd == "run_everything":
assert TRACK_ROUTE
targets = [1040]
for (feed, feed_dict) in FEED_LIST:
#if feed == "old_daily":
# continue
print("feed: ", feed)
a2a_pipeline = AllToAllRoutingPipeline(feed_dict, routing_params(feed))
if True:
a2a_pipeline.loop_trough_targets_and_run_routing_with_route(targets, "testing")
a2a_pipeline.store_pickle_in_db()
a2a_pipeline.calculate_additional_columns_for_journey()
elif cmd == "diff_db":
tables = ["n_boardings",
"journey_duration",
"in_vehicle_duration",
"transfer_wait_duration",
"walking_duration",
"pre_journey_wait_fp",
"temporal_distance"]
diff = DiffDataManager(DIFF_PATH)
diff.initialize_journey_comparison_tables(tables, (FEED_DICT["old_daily"]["journey_dir"], "old_daily"),
(FEED_DICT["lm_daily"]["journey_dir"], "lm_daily"))
elif cmd == "to_db":
a2a_pipeline = AllToAllRoutingPipeline()
a2a_pipeline.store_pickle_in_db()
elif cmd == "post_import":
a2a_pipeline = AllToAllRoutingPipeline()
a2a_pipeline.calculate_additional_columns_for_journey()
elif cmd == "od_measures":
a2a_pipeline = AllToAllRoutingPipeline()
a2a_pipeline.calculate_comparison_measures()