def get_output_dict_with_less_parameter(predict_speed_dict, target_dt, time_slot_interval):
save_filename_list = ["way_types", "way_type_avg_speed_limit", "final_way_table", "final_node_table"]
temp_map_dates = graph_reader(Path("graph/"), SAVE_TYPE_PICKLE, save_filename_list)
way_types = temp_map_dates[0]
way_type_avg_speed_limit = temp_map_dates[1]
final_way_table = temp_map_dates[2]
final_node_table = temp_map_dates[3]
interval_idx = (target_dt.hour * 60 + target_dt.minute) // time_slot_interval
return get_output_dict(predict_speed_dict, target_dt, time_slot_interval, interval_idx,final_node_table,
final_way_table, way_types, way_type_avg_speed_limit)
def generate_way_structure_json(save_path="static/mapdata/way_structure.json"):
save_filename_list = ["final_node_table", "final_way_table"]
temp_map_dates = graph_reader(Path("graph/"), SAVE_TYPE_PICKLE, save_filename_list)
final_node_table = temp_map_dates[0]
final_way_table = temp_map_dates[1]
way_structure = {}
for way, waypoints in final_way_table.items():
points = []
for waypoint in waypoints:
points.append([final_node_table[waypoint][0], final_node_table[waypoint][1]])
way_structure[way] = points
temp_filepath = Path(save_path)
temp_filepath.parent.mkdir(parents=True, exist_ok=True)
with open(temp_filepath, 'w') as f:
# json.dump(output_dict, f, indent=2)
json.dump(way_structure, f)
return 0
def show_traffic_speed(final_way_table, final_node_table, road_speeds, time_range_start_index,
time_range_end_index, time_slot_interval, map_type="OSM"):
"""
Parameters
----------
final_node_table: Dictionary
A dictionary that stored the node id and the latitude/longitude coordinates as a key value pair.
final_way_table: Dictionary
A dictionary that stored the way id and a list of node id's as a key value pair.
road_speeds : Dictionary
A dictionay that stored the [speed matrix] use way id as key and use list to store each row
time_range_start_index : int
The index of the start of the time range
time_range_end_index : int
The index of the end of the time range (include the end time range)
time_slot_interval : int
The length of each time interval in minutes. The input number should be divisible by 1440 (24 hour * 60 min)
by default it is 5 min
map_type : str, optional
The map platform of the generate map. Could be GoogleMap or OSM, by default "OSM"
Returns
-------
None
"""
save_filename_list = ["way_types", "way_type_avg_speed_limit"]
temp_map_dates = graph_reader(Path("graph/"), SAVE_TYPE_PICKLE, save_filename_list)
way_types = temp_map_dates[0]
way_type_avg_speed_limit = temp_map_dates[1]
if map_type == "GoogleMap":
return show_traffic_speed_googlemap(final_way_table, final_node_table, road_speeds, time_range_start_index,
time_range_end_index, time_slot_interval, way_types,
way_type_avg_speed_limit)
elif map_type == "OSM":
return show_traffic_speed_OSM(final_way_table, final_node_table, road_speeds, time_range_start_index,
time_range_end_index, time_slot_interval, way_types, way_type_avg_speed_limit)
else:
print("Unknown map_type")
return 0
def show_traffic_speed(road_speeds, timestamp, map_type="OSM"):
graph_need_read = [
"way_types", "way_type_avg_speed_limit", "final_node_table",
"final_way_table"
]
temp_map_dates = graph_reader(Path("graph"), SAVE_TYPE_PICKLE,
graph_need_read)
way_types = temp_map_dates[0]
way_type_avg_speed_limit = temp_map_dates[1]
final_node_table = temp_map_dates[2]
final_way_table = temp_map_dates[3]
if map_type == "GoogleMap":
return show_traffic_speed_googlemap(final_way_table, final_node_table,
road_speeds, timestamp)
elif map_type == "OSM":
return show_traffic_speed_OSM(final_way_table, final_node_table,
road_speeds, timestamp, way_types,
way_type_avg_speed_limit)
else:
print("Unknown map_type")
return 0
def generate_prediction_in_large_batches(predict_timestamp=int(datetime.now().timestamp()), interval=5,
result_file_path="data/{0}/result/{0}_{1}_min_road.csv",
config_history_date=PREDICT_ROAD_CONDITION_CONFIG_HISTORY_DATE,
config_history_data_range=PREDICT_ROAD_CONDITION_CONFIG_HISTORY_DATA_RANGE,
config_weight=PREDICT_ROAD_CONDITION_CONFIG_WEIGHT):
"""
Generate prediction in large batches (in day(s))
predict_timestamp: int (timestamp)
10-digit timestamp, use current time if not provided
This timestamp will be use to get the date ONLY
interval: int
The length of each time interval in minutes. The input number should be divisible by 1440 (24 hour * 60 min)
by default it is 5 min.
config_history_date: List of int
This parameter specifies which historical days of data the function needs to use in the computation. It should
be a List of int, where each int means the offset of the day that need predict. e.g. -1 means yesterday
by default it will use a predetermined configuration
config_history_data_range: List of int
This parameter specifies the range and the order of using the nearby data to replace the missing data. If the
value is [-1, 1] and if the data we are looking for located on the ith index is missing. We will try to use the
value at i-1 or i+1 as the data located in ith index.
by default it will use a predetermined configuration
config_weight: List of float
This parameter specifies the weight of each day's data when compute the weighted sum.
by default it will use a predetermined configuration
Returns
-------
None
"""
# Check input, load data and preparation
if len(config_history_date) != len(config_weight):
print("error: len(config_history_date) != len(config_weight)")
return -1
save_filename_list = ["way_graph", "way_types", "way_type_avg_speed_limit", "final_node_table", "final_way_table"]
temp_map_dates = graph_reader(Path("graph/"), SAVE_TYPE_PICKLE, save_filename_list)
way_graph = temp_map_dates[0]
way_types = temp_map_dates[1]
way_type_avg_speed_limit = temp_map_dates[2]
final_node_table = temp_map_dates[3]
final_way_table = temp_map_dates[4]
# Find the time of the predict, also find the range index in the day.
predict_time = datetime.fromtimestamp(predict_timestamp)
predict_time_date_str = predict_time.strftime("%Y%m%d")
# Prepare of load history data
history_data_date_str = []
for offset in config_history_date:
history_data_date_str.append((predict_time + timedelta(days=offset)).strftime("%Y%m%d"))
if FLAG_DEBUG:
print("Predict time: {}".format(predict_time.strftime("%Y-%m-%d %H:%M:%S")))
print("History data date_str:{}".format(history_data_date_str))
# Load history data
history_speed_matrix_list, config_weight = \
predict_road_condition.get_history_speed_matrix_list(history_data_date_str, config_weight, interval,
result_file_path)
if len(history_speed_matrix_list) == 0:
print({"Error": "No enough data for predict"})
return {"Error": "No enough data for predict"}
if FLAG_DEBUG:
print("{} day(s) load".format(len(history_speed_matrix_list)))
full_way_id_set, usable_way_id_set = predict_road_condition.get_way_id_set(history_speed_matrix_list)
for interval_idx in tqdm(range(int(1440 / interval))):
# print(interval_idx, time_range_index_to_str(interval_idx, interval))
predict_speed_dict = \
predict_road_condition.compute_speed_dict(interval, interval_idx, history_speed_matrix_list,
full_way_id_set,
usable_way_id_set, config_history_data_range, config_weight,
way_graph, way_types, way_type_avg_speed_limit)
predict_speed_dict_to_json(predict_speed_dict, predict_time, interval, interval_idx,
final_node_table, final_way_table, way_types, way_type_avg_speed_limit)
return 0
save_type = SAVE_TYPE_JSON
elif sys.argv[4] == "pickle":
save_type = SAVE_TYPE_PICKLE
else:
print("invalid Save format")
print(
"Save format: the format to save the result, by default is pickle"
)
print(" possible value: JSON and pickle")
exit(0)
print("datapoint : %s" % datapoint)
print("Result path: %s" % result_file_path)
if save_type == SAVE_TYPE_PICKLE:
print("Result type: pickle")
elif save_type == SAVE_TYPE_JSON:
print("Result type: JSON")
save_filename_list = [
"final_node_table", "final_way_table", "final_relation_table"
]
map_dates = graph_reader(result_file_path, save_type, save_filename_list)
final_node_table = map_dates[0]
final_way_table = map_dates[1]
final_relation_table = map_dates[2]
# 9345830 is 35A
find_nearest_road(final_node_table, final_way_table, final_relation_table,
[9345830], datapoint)
from datetime import datetime
from pathlib import Path
from helper.global_var import SAVE_TYPE_PICKLE
from helper.graph_reader import graph_reader
if __name__ == '__main__':
date_str = datetime.today().strftime('%Y%m%d')
# date_str = "20220131"
data_root = Path(".") / 'data'
process_data.preprocess_data(date_str, overwrite=True, min_file_size=10)
reformat_data.reformat_by_bus(date_str)
reformat_data.sort_reformat_data(date_str)
save_filename_list = [
"final_node_table", "final_way_table", "final_relation_table"
]
map_dates = graph_reader(Path("graph"), SAVE_TYPE_PICKLE,
save_filename_list)
final_node_table = map_dates[0]
final_way_table = map_dates[1]
final_relation_table = map_dates[2]
time_slot_intervals = [5, 15]
for interval in time_slot_intervals:
print("Processing interval:", interval)
find_traffic_speed.find_traffic_speed(date_str,
final_node_table,
final_way_table,
final_relation_table,
time_slot_interval=interval)
def retrieve_traffic_data(timestamp, time_interval):
"""
Get the traffic data for the given timestamp (at the nearest time interval/slot) and time_interval.
Parameters
----------
timestamp: str
A 10 digit timestamp
time_interval: str
The time interval size that the user has selected, e.g., 15, 30, and 45.
Returns
-------
Serialized Json object, containing traffic data, which has the following format,
{
"generate_timestr": "2021-03-02 19:52:44",
"generate_timestamp": 1614732764, .
"time_slot_interval": 15,
"interval_idx": 0,
"predict_time_range": "2020-07-30 00:00 - 00:14",
"road_speed": {way_id: {"speed": 16.274295463111194,
"speed_ratio": 0.46497987037460553},
...
}
}
"""
# retrieve traffic at the specific timestamp (during the nearest interval)
# print(timestamp)
dt_target = datetime.fromtimestamp(int(timestamp))
dt_now = datetime.now()
time_interval = int(time_interval)
dt_diff_min = int((dt_target - dt_now).seconds / 60)
dt_diff_day = int((dt_target.date() - dt_now.date()).days)
if dt_diff_day >= 0:
if dt_diff_min <= 120:
# Less then 2 hours
return jsonify({"error": "no_data"})
else:
# find the nearest interval of the timestamp based on the time_interval size
interval_idx = get_nearest_interval(dt_target, time_interval)
data_path = Path(
GPILB_CACHE_PATH.format(dt_target.strftime('%Y%m%d'),
time_interval, interval_idx))
if data_path.is_file():
with open(data_path, 'r') as fp:
data = json.load(fp)
else:
predict_speed_dict = predict_road_condition.predict_road_condition(
dt_target.timestamp(), interval=int(time_interval))
data = predict_result_helper.get_output_dict_with_less_parameter(
predict_speed_dict, dt_target, time_interval)
return jsonify(data) # serialize and use JSON headers
else:
# Past time, can use existing data
interval_idx = get_nearest_interval(dt_target, time_interval)
date_str = dt_target.strftime("%Y%m%d")
temp_filepath_csv = Path("data/{0}/result/{0}_{1}_min_road.csv".format(
date_str, time_interval))
temp_filepath_p = temp_filepath_csv.with_suffix('.p')
speed_matrix = {}
if temp_filepath_p.is_file():
with open(temp_filepath_p, 'rb') as f:
speed_matrix = pickle.load(f)
elif temp_filepath_csv.is_file():
speed_matrix = predict_road_condition.read_speed_matrix_from_file(
temp_filepath_csv)
else:
return jsonify({"error": "no_data"})
predict_speed_dict = {}
for way_id, his_speeds in speed_matrix.items():
predict_speed_dict[way_id] = his_speeds[interval_idx]
save_filename_list = [
"way_graph", "way_types", "way_type_avg_speed_limit"
]
temp_map_dates = graph_reader(Path("graph/"), SAVE_TYPE_PICKLE,
save_filename_list)
way_graph = temp_map_dates[0]
way_types = temp_map_dates[1]
way_type_avg_speed_limit = temp_map_dates[2]
predict_speed_dict = predict_road_condition.estimate_no_data_road_speed_using_BFS(
predict_speed_dict, way_graph, way_types, way_type_avg_speed_limit)
data = predict_result_helper.get_output_dict_with_less_parameter(
predict_speed_dict, dt_target, time_interval)
return jsonify(data)
def predict_road_condition(predict_timestamp=int(datetime.now().timestamp()), interval=5,
result_file_path="data/{0}/result/{0}_{1}_min_road.csv",
config_history_date=PREDICT_ROAD_CONDITION_CONFIG_HISTORY_DATE,
config_history_data_range=PREDICT_ROAD_CONDITION_CONFIG_HISTORY_DATA_RANGE,
config_weight=PREDICT_ROAD_CONDITION_CONFIG_WEIGHT):
"""
This function will reading historical data and using weighted sum calculate the bus speed on the road at the
given time.
Parameters
----------
predict_timestamp: timestamp
10-digit timestamp, use current time if not provided
interval: int
The length of each time interval in minutes. The input number should be divisible by 1440 (24 hour * 60 min)
by default it is 5 min.
result_file_path: String
The path (also the format) to the result .csv files.
{0} is a 8-digit date_str in yyyyMMdd format.
{1} is the value of interval
by default it will use the project's file format.
config_history_date: List of int
This parameter specifies which historical days of data the function needs to use in the computation. It should
be a List of int, where each int means the offset of the day that need predict. e.g. -1 means yesterday
by default it will use a predetermined configuration
config_history_data_range: List of int
This parameter specifies the range and the order of using the nearby data to replace the missing data. If the
value is [-1, 1] and if the data we are looking for located on the ith index is missing. We will try to use the
value at i-1 or i+1 as the data located in ith index.
by default it will use a predetermined configuration
config_weight: List of float
This parameter specifies the weight of each day's data when compute the weighted sum.
by default it will use a predetermined configuration
Returns
-------
predict_speed_dict: Dictionary
A dictionary that use way_id as key and the predict bus speed on that road at the given time as value.
When there is an error, it will return a dictionary with key "Error" and the detail of the error as the value
"""
# TODO: This function could be implemented using pandas and numpy for better performance(Shiluo)
# Check input, load data and preparation
if len(config_history_date) != len(config_weight):
return -1
save_filename_list = ["way_graph", "way_types", "way_type_avg_speed_limit"]
temp_map_dates = graph_reader(Path("graph/"), SAVE_TYPE_PICKLE, save_filename_list)
way_graph = temp_map_dates[0]
way_types = temp_map_dates[1]
way_type_avg_speed_limit = temp_map_dates[2]
# Find the time of the predict, also find the range index in the day.
predict_time = datetime.fromtimestamp(predict_timestamp)
# predict_time_date_str = predict_time.strftime("%Y%m%d")
interval_idx = math.floor((predict_time.hour * 3600 + predict_time.minute * 60 + predict_time.second) /
(interval * 60))
# Prepare of load history data
history_data_date_str = []
for offset in config_history_date:
history_data_date_str.append((predict_time + timedelta(days=offset)).strftime("%Y%m%d"))
if FLAG_DEBUG:
print("Predict time: {}".format(predict_time.strftime("%Y-%m-%d %H:%M:%S")))
print("History data date_str:{}".format(history_data_date_str))
# Load history data
history_speed_matrix_list, config_weight = get_history_speed_matrix_list(history_data_date_str, config_weight,
interval, result_file_path)
if len(history_speed_matrix_list) == 0:
return {"Error": "No enough data for predict"}
if FLAG_DEBUG:
print("{} day(s) load".format(len(history_speed_matrix_list)))
full_way_id_set, usable_way_id_set = get_way_id_set(history_speed_matrix_list)
predict_speed_dict = compute_speed_dict(interval, interval_idx, history_speed_matrix_list, full_way_id_set,
usable_way_id_set, config_history_data_range, config_weight,
way_graph, way_types, way_type_avg_speed_limit)
if FLAG_DEBUG:
print(show_traffic_speed(predict_speed_dict, predict_timestamp))
# print(predict_speed_dict)
return predict_speed_dict