def _generate_weighted_edges(self, weight_matrices, time_interval_start):
log("Generating weighted edges...")
if time_interval_start not in self.avail_time_intervals:
raise Exception("The given time interval is not valid. Did you check available time intervals?")
# Get index of given time interval.
time_idx = None
for i, avail_time in enumerate(self.avail_time_intervals):
if avail_time == time_interval_start:
log("Found requested time interval")
time_idx = i
break
weight_matrix = weight_matrices[time_idx]
result = WeightResult()
result.time_interval_start = time_interval_start
result.speed_intervals = [0, 5, 10, 15, 20, 25, 30, 35, 40]
weighted_edges = []
for i, edge_id in enumerate(self._edges_lst):
w_edge = WeightedEdge()
# Map edge.
w_edge.edge_id = edge_id
node_ids = edge_id.split('-')
w_edge.start_node_osm_id = node_ids[0]
w_edge.end_node_osm_id = node_ids[1]
# Get weights from this edge in input matrix!
w_edge.weights = weight_matrix[i].tolist()
weighted_edges.append(w_edge)
result.weighted_edges = weighted_edges
return result
def _load_data(self, pickle_loader):
self._edges_lst = pickle_loader.load_edges()
self._input_matrices = pickle_loader.load_input_matrices()
self._prediction_matrices = pickle_loader.load_prediction_matrices()
self._time_intervals_srs = pickle_loader.load_time_intervals()
# Set global index limit to num of indices in prediction results.
self._limit = len(self._prediction_matrices)
log("Pickles loaded.")
def main(_):
log("In gcrn_main_gcnn")
prepare_dirs(config)
prepare_config_date(config, config.ds_ind)
# Random seed settings
rng = np.random.RandomState(config.random_seed)
tf.set_random_seed(config.random_seed)
# Model training
log("Initilializing trainer...")
trainer = Trainer(config, rng)
save_config(config.model_dir, config)
log("Trainer initialized!")
config.load_path = config.model_dir
if config.is_train:
log("Training...")
trainer.train(save=True)
log("Training done!")
log("Testing...")
result_dict = trainer.test()
log("Testing done!")
else:
if not config.load_path:
raise Exception(
"[!] You should specify `load_path` to "
"load a pretrained model")
log("Testing...")
result_dict = trainer.test()
log("Testing done!")
save_results(config.result_dir, result_dict)
accept_rate = evaluate_result(result_dict, method='KS-test', alpha=0.1)
kl_div = evaluate_result(result_dict, method='KL')
wasser_dis = evaluate_result(result_dict, method='wasser')
sig_test = evaluate_result(result_dict, method='sig_test')
print("The accept rate of KS test is ", accept_rate)
print("The final KL div is ", kl_div)
print("The wasser distance is ", wasser_dis)
print("The AR of Sign Test is ", sig_test)
# Please note: Files placed in the package "graphcompletionlib" are a clone of the GraphCompletion algorithm
# found here: https://github.com/hujilin1229/GraphCompletion
# However, a few modifications has been made to the algorithm in order to let adapt it to this project.
# These modifications are marked by "SW502" as far as possible.
from datetime import datetime
from flask import Flask
from flask_cors import CORS
import sys
from main.common.logger import log
from main.configuration.config import *
from main.core.models import *
from graphcompletionlib import gcrn_main_gcnn
from main.core.services import WeightCompleter
log("Starting app...")
if len(sys.argv) > 1 and sys.argv[1] == "prod":
print("Loading production config")
config = ProductionConfig()
else:
print("Loading development config")
config = DevelopmentConfig()
app = Flask(__name__)
app.config.from_object(config)
log(f'ENV is set to: {app.config["ENV"]}')
from main.web.routes import routes
app.register_blueprint(routes)
def __init__(self,
sname,
mode,
target,
sample_rate,
win_size,
hist_range,
s_month,
s_date,
e_month,
e_date,
data_rm,
batch_size=-1,
coarsening_level=4,
conv_mode='gcnn',
is_coarsen=True,
is_predicting=False):
base_dir = os.path.join('.\graphcompletionlib', 'data', sname)
print(base_dir)
if target == 'avg':
data_dir = os.path.join(
base_dir, '{}_{}'.format(sample_rate, win_size), mode, target,
'{}_{}-{}_{}'.format(s_date, s_month, e_date,
e_month), 'rm{}'.format(data_rm))
else:
data_dir = os.path.join(
base_dir, '{}_{}'.format(sample_rate, win_size), mode, target,
'{}_{}_{}'.format(hist_range[0], hist_range[-1] + 1,
hist_range[1] - hist_range[0]),
'{}_{}-{}_{}'.format(s_date, s_month, e_date,
e_month), 'rm{}'.format(data_rm))
dict_normal_fname = os.path.join(data_dir, 'dict_normal.pickle')
train_data_dict_fname = os.path.join(data_dir,
'train_data_dict.pickle')
validate_data_dict_fname = os.path.join(data_dir,
'validate_data_dict.pickle')
Adj_fname = os.path.join(base_dir, 'edge_adj.pickle')
if not os.path.exists(dict_normal_fname) or \
not os.path.exists(train_data_dict_fname) or \
not os.path.exists(validate_data_dict_fname) or \
not os.path.exists(Adj_fname):
print("Data does not exist! Creating datasets...")
self.data_generator(base_dir, data_dir, sname, mode, target,
sample_rate, win_size, hist_range, s_month,
s_date, e_month, e_date, data_rm,
is_predicting)
print("Data exists! Loading datasets...")
adj = pklLoad(Adj_fname)
dict_normal = pklLoad(dict_normal_fname)
train_data_dict = pklLoad(train_data_dict_fname)
validate_data_dict = pklLoad(validate_data_dict_fname)
log("Data loaded")
if target == 'avg':
self.y_scaler = dict_normal['velocity']
else:
self.y_scaler = None
train_data = train_data_dict['velocity_x']
train_labels = train_data_dict['velocity_y']
train_label_weight = train_data_dict['weight_y']
train_counts = train_data_dict['count_y']
train_vel_lists = train_data_dict['vel_list']
cat_train = train_data_dict['cat']
con_train = train_data_dict['con']
log(f"Train data loaded, sample count: {len(train_data)} ")
test_data = validate_data_dict['velocity_x']
test_labels = validate_data_dict['velocity_y']
test_labels_weight = validate_data_dict['weight_y']
test_counts = validate_data_dict['count_y']
test_vel_lists = validate_data_dict['vel_list']
cat_test = validate_data_dict['cat']
con_test = validate_data_dict['con']
log(f"Test data loaded, sample count: {len(test_data)} ")
# self.mean_y = mean_gt(train_data)
# self.mean_y[np.isnan(self.mean_y)] = 0.0
# train_data = fill_mean(train_data, self.mean_y)
# train_labels = fill_mean(train_labels, self.mean_y)
# train_label_weight = np.ones(train_label_weight.shape)
#
# test_data = fill_mean(test_data, self.mean_y)
if conv_mode == 'gcnn':
perm_file = os.path.join(base_dir, 'adj_perm.pickle')
graph_file = os.path.join(base_dir, 'perm_graphs.pickle')
if os.path.exists(perm_file) and os.path.exists(graph_file):
self.perm = pklLoad(perm_file)
self.graphs = pklLoad(graph_file)
else:
self.graphs, self.perm = coarsening.coarsen(
adj, levels=coarsening_level, self_connections=False)
pklSave(perm_file, self.perm)
pklSave(graph_file, self.graphs)
if is_coarsen:
train_data = coarsening.perm_data_hist(train_data, self.perm)
test_data = coarsening.perm_data_hist(test_data, self.perm)
self.all_batches = []
self.sizes = []
log(f"Constrcuting batches... Batch size set to {batch_size}.")
######################
# SW502 Modification
######################
if (is_predicting):
log("Constructing only test batches...")
self.construct_batches_all_test(test_data, test_labels,
test_labels_weight, cat_test,
con_test, test_counts,
test_vel_lists, batch_size)
# End of mod.
else:
log("Splitting batches into train, val, tests... ")
val_x, test_x, val_y, test_y, \
val_y_weight, test_y_weight, \
val_con, test_con, val_cat, \
test_cat, val_count, test_count, \
val_vel_list, test_vel_list = \
train_test_split(test_data, test_labels, test_labels_weight,
con_test, cat_test, test_counts,
test_vel_lists, test_size=0.8)
print("Reshaping tensors...")
# Split train, val, tests data into batches
self.construct_batches(train_data, train_labels,
train_label_weight, cat_train, con_train,
train_counts, train_vel_lists, batch_size)
self.construct_batches(val_x, val_y, val_y_weight, val_cat,
val_con, val_count, val_vel_list,
batch_size)
self.construct_batches(test_x, test_y, test_y_weight, test_cat,
test_con, test_count, test_vel_list,
batch_size)
print(
"Data load done. Number of batches in train: %d, val: %d, test: %d"
% (self.sizes[0], self.sizes[1], self.sizes[2]))
self.adj = adj
self.batch_idx = [0, 0, 0]