triplet_loss = TripletLoss(G, False)
indices = []
losses = []
def train(epoch):
with torch.autograd.set_detect_anomaly(True):
t = time.time()
model.train()
optimizer.zero_grad()
output = model(features, adj_train)
# triplet_loss.embeddings = output
loss = triplet_loss._batch_hard_triplet_loss(output)
# loss = triplet_loss.get_loss_margin()
losses.append(loss)
indices.append(epoch)
loss.backward()
optimizer.step()
print('Epoch: {:04d}'.format(epoch + 1),
'loss_train: {:.4f}'.format(loss.item()),
'time: {:.4f}s'.format(time.time() - t))
for epoch in range(args.epochs):
train(epoch)
torch.save(model.state_dict(), "models/gcn_hom_onto2")
plot_loss(indices, losses, "Epooch", "Loss", "Epooch vs loss -- gcn ontology")
history = pretrained_model.fit_generator(
train_gen,
steps_per_epoch=nbatches_train,
epochs=num_epochs,
validation_data=validation_gen,
validation_steps=nbatches_validation,
callbacks=[early_stopping, checkpoint])
else:
if use_class_weights:
history = pretrained_model.fit_generator(
train_gen,
steps_per_epoch=nbatches_train,
epochs=num_epochs,
class_weight=class_weights,
validation_data=validation_gen,
validation_steps=nbatches_validation,
callbacks=[early_stopping])
else:
history = pretrained_model.fit_generator(
train_gen,
steps_per_epoch=nbatches_train,
epochs=num_epochs,
validation_data=validation_gen,
validation_steps=nbatches_validation,
callbacks=[early_stopping])
# plot accuracy and loss for train and validation
if num_epochs > 1:
plots.plot_accuracy(figures_dir, 'accuracy_epochs_vgg16', history)
plots.plot_loss(figures_dir, 'loss_epochs_vgg16', history)
def main():
'''
This is the MCENET model for trajectory prediction.
It leverages scene, occupancy grid, and trajectry sequence for trajectory prediction
Training:
X-Encoder for encoding the information from observation
Y-Encoder for encoding the information from ground truth
Both encoded information are concatenated for parameterizing the latent variable z,
which is pushed towards a Gaussian distribution
z and the encoded information from X-Encoder is used as the condition for reconstructing the future trajectory
Inference:
X-Encoder for encoding the information from observation
Y-Encoder is removed
z is sampled from the Gaussian distribution
z and the encoded information from X-Encoder is used as the condition for predicting the future trajectory
'''
timestr = time.strftime("%Y%m%d-%H%M%S")
dataname = 'HC'
print(dataname)
# Make all the necessary folders
mak_dir()
bg_image, data_dir, dirs, filename = get_data_repo(dataname)
args = parse_args(dataname)
# # Generate data
if args.data_process == True:
generate_data(dataname, leaveoneout=args.leaveoneout)
# specify which GPU(s) to be used, gpu device starts from 0
# os.environ["CUDA_VISIBLE_DEVICES"] = "0"
# # Use the default CPU
# os.environ["CUDA_VISIBLE_DEVICES"] = ""
# CHECK POINT AND SAVE THE BEST MODEL
filepath = "../models/mcenet_mixed_%s_%0.f_%s.hdf5" % (
dataname, args.epochs, timestr)
earlystop = EarlyStopping(monitor='val_loss',
mode='min',
verbose=1,
patience=50)
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=0,
save_best_only=True,
mode='min')
callbacks_list = [earlystop, checkpoint]
# INSTANTIATE THE MODEL
mcenet = MCENET(args)
# Construct the training model
train = mcenet.training()
train.summary()
# Load the test data
# Note the one step difference between position and offset
obs_seq = args.obs_seq - 1
#################### START TRAINING THE MCENet MODEL ####################
if args.train_mode:
print("\nload the training data")
T_ = np.load('../processed/%s_training_data_grid.npz' % dataname)
obs, pred, obs_og, pred_og, obs_hmaps, pred_hmaps = T_['obs'], T_[
'pred'], T_['obs_og'], T_['pred_og'], T_['obs_hmaps'], T_[
'pred_hmaps']
print('You are using the scene context from %s' % args.sceneType)
print('Data loaded!')
# Shift one time step for occupancy grid and scene attention
obs_og = obs_og[:, 1:, :]
obs_hmaps = obs_hmaps[:, 1:, ...]
# Get the residual of the trajectories
traj = np.concatenate((obs[:, :, 2:4], pred[:, :, 2:4]), axis=1)
traj_r = traj[:, 1:, :] - traj[:, :-1, :]
traj_r = traj_r * args.resi_scale
obs_r, pred_r = traj_r[:, :obs_seq, :], traj_r[:, obs_seq:, :]
## Get User type one-hot encoding
obs_type = get_type(obs, 3, isObservation=True)
obs_r = np.concatenate((obs_r, obs_type), axis=-1)
obs_hpinput = mobilenet(obs_hmaps, obs_seq)
pred_hpinput = mobilenet(pred_hmaps, args.pred_seq)
# Here is the training related data
print('Check the input data...')
print('the shape of obs', obs_r.shape)
print('the shape of pred', pred_r.shape)
#print('the number of road users in train_raw', len(np.unique(train_raw[:, 0])))
print('the shape of obs_og', obs_og.shape)
print('the shape of pred_og', pred_og.shape)
print('the shape of obs_maps', obs_hmaps.shape)
print('the shape of pred_maps', pred_hmaps.shape)
# Get the data fro training and validation
np.random.seed(10)
train_val_split = np.random.rand(len(traj)) < args.split
train_obs_hpinput = obs_hpinput[train_val_split]
train_pred_hpinput = pred_hpinput[train_val_split]
train_obs_og = obs_og[train_val_split]
train_pred_og = pred_og[train_val_split]
train_obs_r = obs_r[train_val_split]
train_pred_r = pred_r[train_val_split]
val_obs_hpinput = obs_hpinput[~train_val_split]
val_pred_hpinput = pred_hpinput[~train_val_split]
val_obs_og = obs_og[~train_val_split]
val_pred_og = pred_og[~train_val_split]
val_obs_r = obs_r[~train_val_split]
val_pred_r = pred_r[~train_val_split]
print('Start training the MCENet model...')
history = train.fit(x=[
train_obs_hpinput, train_pred_hpinput, train_obs_og, train_pred_og,
train_obs_r, train_pred_r
],
y=train_pred_r,
shuffle=True,
epochs=args.epochs,
batch_size=args.batch_size,
verbose=1,
callbacks=callbacks_list,
validation_data=([
val_obs_hpinput, val_pred_hpinput, val_obs_og,
val_pred_og, val_obs_r, val_pred_r
], val_pred_r))
print('Training the MCENet model done!')
print('Plotting loss...')
plot_loss(history, args.scale, args.real_scale, timestr, dataname)
print('Plotting done!')
### Here load the best trained model
train.load_weights(filepath)
else:
print("\nLoad the trained model")
if args.sceneType == 'heatmap':
print("The scene context is heatmap")
trained_model = "../trained_model/....hdf5"
elif args.sceneType == "aerial_photograph":
print("The scene context is aerial photograph")
trained_model = "../trained_model/....hdf5"
elif args.sceneType == 'segmented_map':
print("The scene context is segmented map")
trained_model = "../models/mcenet_mixed_HC_2000_20200618-213259.hdf5"
train.load_weights(trained_model)
#################### START TESTING ####################
# NOTE THAT IN TESTING PHASE, ONLY x (observed trajectory) is available
# construct the encoder to get the x_encoded_dense, including scene, occupancy, and trajectory sequence information
print('Start testing...')
print('Load the test data ...')
test_ = np.load('../processed/%s_test_data_grid.npz' % dataname)
test_obs, test_obs_og, test_obs_hmaps = test_['obs'], test_[
'obs_og'], test_['obs_hmaps']
# Shift one time step for occupancy grid and scene attention
test_obs_og = test_obs_og[:, 1:, :]
test_obs_hmaps = test_obs_hmaps[:, 1:, ...]
# Get the residual of the trajectories
test_obs_r = test_obs[:, 1:, 2:4] - test_obs[:, :-1, 2:4]
test_obs_r = test_obs_r * args.resi_scale
## Get User type one-hot encoding
test_obs_type = get_type(test_obs, 3, isObservation=True)
test_obs_r = np.concatenate((test_obs_r, test_obs_type), axis=-1)
# Construct the heatmap/scene inputs for the model
test_obs_hpinput = mobilenet(test_obs_hmaps, obs_seq)
# Double check the test data shape
print('the shape of test_obs', test_obs_r.shape)
print('the shape of test_obs_og', test_obs_og.shape)
print('the shape of test_obs_hmaps', test_obs_hmaps.shape)
# Retrieve the x_encoder and the decoder
x_encoder = mcenet.X_encoder()
generator = mcenet.Decoder()
# get the x_encoded_dense as latent feature for prediction
x_latent = x_encoder.predict([test_obs_hpinput, test_obs_og, test_obs_r],
batch_size=args.batch_size)
# Save the summary of the model
with open(
'../models/mcenet_mixed_%s_model_summary_%s.txt' %
(dataname, timestr), 'w') as f:
with redirect_stdout(f):
train.summary()
x_encoder.summary()
generator.summary()
# START PREDICTING USING THE x_encoded_dense AND SAMPLED LATENT VARIABLE z
print('Start predicting...')
### Change the residual back to positions
last_obs_test = test_obs[:, -1, 2:4]
predictions = []
for i, x_ in enumerate(x_latent):
last_pos = last_obs_test[i]
x_ = np.reshape(x_, [1, -1])
for i in range(args.num_pred):
# sampling z from a normal distribution
z_sample = np.random.rand(1, args.z_dim)
y_p = generator.predict(np.column_stack([z_sample, x_]))
y_p = y_p / args.resi_scale
y_p_ = np.concatenate(([last_pos], np.squeeze(y_p)), axis=0)
y_p_sum = np.cumsum(y_p_, axis=0)
predictions.append(y_p_sum[1:, :])
predictions = np.reshape(predictions,
[-1, args.num_pred, args.pred_seq, 2])
test_pred, test_raw = test_['pred'], test_['raw']
indexed_predictions = get_index_prediction(test_obs, test_pred,
predictions, args.num_pred,
args.pred_seq)
print('Predicting done!')
## this is the error for the sampling (aveage and minimum)
classfied_errors = get_classified_errors(test_pred, indexed_predictions,
args.scale / args.real_scale)
## Select the most likely prediction based on Gaussian rank
selected_preds = np.zeros((0, 5))
for pred in indexed_predictions:
select_index = gauss_rank(pred)
for m, traj in enumerate(pred):
if m == select_index:
selected_preds = np.vstack((selected_preds, traj))
selected_preds = selected_preds.reshape(indexed_predictions.shape[0], 1,
indexed_predictions.shape[2],
indexed_predictions.shape[3])
selected_errors = get_classified_errors(test_pred, selected_preds,
args.scale / args.real_scale)
np.savetxt('../results/mcenet_mixed_%s_statistic_results_%.2f_%s.txt' %
(dataname, classfied_errors[0, 2], timestr),
classfied_errors,
delimiter=',')
np.savetxt(
'../selected_results/mcenet_mixed_%s_statistic_results_%.2f_%s.txt' %
(dataname, selected_errors[0, 2], timestr),
selected_errors,
delimiter=',')
# Save the predictions
print('Start saving the predictions...')
np.save(
'../predictions/mcenet_mixed_%s_test_obs_grid_%.2f_%s.npy' %
(dataname, classfied_errors[0, 2], timestr), test_obs)
np.save(
'../predictions/mcenet_mixed_%s_test_gt_grid_%.2f_%s.npy' %
(dataname, classfied_errors[0, 2], timestr), test_pred)
np.save(
'../predictions/mcenet_mixed_%s_test_predictions_grid_%.2f_%s.npy' %
(dataname, classfied_errors[0, 2], timestr), indexed_predictions)
print('Results saved...')
# Plotting the predicted trajectories
print('Start plotting the predictions...')
plot_dir = '../plots/mcenet_mixed_%s_plot_%.2f_%s' % (
dataname, classfied_errors[0, 2], timestr)
if not os.path.exists(plot_dir):
os.mkdir(plot_dir)
print("Directory mcenet_mixed_%s_plot_%.2f_%s" %
(dataname, classfied_errors[0, 2], timestr))
else:
print("Directory mcenet_mixed_%s_plot_%.2f_%s" %
(dataname, classfied_errors[0, 2], timestr))
plot_scenarios(test_raw, test_obs, test_pred, indexed_predictions,
bg_image, args.scale, args.real_scale, plot_dir)
print('Plotting Done!')
# Save the model configuration
with open(
'../hyperparameters/mcenet_mixed_%s_args_%.2f_%s.txt' %
(dataname, classfied_errors[0, 2], timestr), 'w') as f:
for arg in vars(args):
params = ('%s = %s' % (str(arg), str(getattr(args, arg))))
f.writelines(params)
f.writelines('\n')
def main():
global args
parser = argparse.ArgumentParser(
description="Convolutional NN Training Script")
parser.add_argument("-c",
"--config",
dest="configfile",
default='config.yml',
help="Path to yaml configuration file")
parser.add_argument("-m",
"--modelnames",
dest="modelnames",
nargs="*",
default=None,
required=False,
help="Model name to test")
args = parser.parse_args()
# Get configuration file
hconfig = ModelConfigurator(args.configfile)
# Extract config parameters
datapath = hconfig.datapath
# Get requested models, if None, take config's list
model_list = args.modelnames
if model_list is None:
model_list = hconfig.avail_models
# List to store histories
hist_list = []
# Loop over requested models
for mod_i in model_list:
mod_i = mod_i.strip()
# Set model config parameters
hconfig.model_config(mod_i)
# Directory structures for model
model_dir_struct = ModelDirStruct(main_dir=hconfig.model_outpath,
test_model=True)
with open(model_dir_struct.hist_file, 'rb') as f:
history = pickle.load(f)
hist_list.append(history)
# Visualize history
plot_accuracy(history=history, model_dir_struct=model_dir_struct)
plot_loss(history=history, model_dir_struct=model_dir_struct)
print('Saved figures to {}'.format(model_dir_struct.plots_dir))
# If more than one model requested, compare validation accuracy
if len(model_list) > 1:
compare_accuracy(names=model_list,
hist_list=hist_list,
model_dir_struct=model_dir_struct)
def train(self,
training_set,
test_set,
nr_of_epochs,
steps_per_epoch,
iFcallbacks=False,
do_plots=False):
if do_plots:
if not iFcallbacks:
raise ValueError(
'Wrong parameters: if wanna make plots iFcallbacks mast be True but is : '
+ str(iFcallbacks))
if iFcallbacks:
csv_logger = create_cv_logger(self.model_name)
history = callback_history()
# earlyStop = callbackEarlyStopping()
checkPoint = callbackCheckpoint(self.model_name)
# tensor = callbackTensor()
callbacks = [csv_logger, history, checkPoint]
history = self.model.fit_generator(
training_set,
steps_per_epoch=steps_per_epoch,
epochs=nr_of_epochs,
validation_data=test_set,
validation_steps=steps_per_epoch / 3,
# use_multiprocessing=True,
workers=6,
callbacks=callbacks)
else:
self.model.fit_generator(
training_set,
steps_per_epoch=steps_per_epoch,
epochs=nr_of_epochs,
validation_data=test_set,
validation_steps=steps_per_epoch / 3,
# use_multiprocessing=True,
workers=6,
)
print('Training is Done!!')
if do_plots:
plot_accuracy(history)
plot_loss(history)
self._save_model()
print('Model is saved')
# cnn.add(conv.ZeroPadding2D( (1,1), input_shape = (1,28,28), ))
### additional parameters for keras model:
# activation = None,
# use_bias = True,
# kernel_initializer = 'glorot_uniform',
# bias_initializer = 'zeros',
# kernel_regularizer = None,
# bias_regularizer = None,
# activity_regularizer = None,
# kernel_constraint = None,
# bias_constraint = None
# from keras import backend as K
# from keras.layers import Layer
#
# class MyLayer():#Layer):
#
# def __init__(self, output_dim, **kwargs):
# self.output_dim = output_dim
# super(MyLayer, self).__init__(**kwargs)
#
# def build(self, input_shape):
# # Create a trainable weight variable for this layer.
# self.kernel = self.add_weight(name='kernel',
# shape=(input_shape[1], self.output_dim),
# initializer='uniform',
# trainable=True)
# super(MyLayer, self).build(input_shape) # Be sure to call this at the end
#
# def call(self, x):
# return K.dot(x, self.kernel)
#
# def compute_output_shape(self, input_shape):
# return (input_shape[0], self.output_dim)
#
# def min_max_pool2d(x):
# max_x = K.pool2d(x, pool_size=(2, 2), strides=(2, 2))
# min_x = -K.pool2d(-x, pool_size=(2, 2), strides=(2, 2))
# return K.concatenate([max_x, min_x], axis=1) # concatenate on channel
#
# def min_max_pool2d_output_shape(input_shape):
# shape = list(input_shape)
# shape[1] *= 2
# shape[2] /= 2
# shape[3] /= 2
# return tuple(shape)
#
# # replace maxpooling layer
# # cnn.add(Lambda(min_max_pool2d, output_shape=min_max_pool2d_output_shape))
tr_bert_classifer)
# initialize net
untrained_net, untrained_net_final = initialize_net(args, device,
tr_bert_classifer)
# get Adam optimzer
adam_optimizer = get_adam_optimzer(untrained_net, args)
# train_net
trained_net, model_results = train_net(dataloaders, adam_optimizer, device,
untrained_net, args, tr_bert_classifer)
#plot figures
plot_accuracies(trained_net.train_acc, trained_net.val_acc, 'all_artists')
plot_loss(trained_net.train_loss, trained_net.val_loss, 'all_artists')
#training final net and predicting test results
print("training final net")
args = update_args_test_predict(args)
del trained_net, adam_optimizer, dataloaders
dataloaders_without_val = get_dataloaders(song_token, embedding_songs, args,
tr_bert_classifer)
adam_optimizer = get_adam_optimzer(untrained_net_final, args)
trained_net, model_results = train_net(dataloaders_without_val, adam_optimizer,
validation_data=validation_gen,
validation_steps=nbatches_validation,
callbacks=[early_stopping, checkpoint])
else:
history = pretrained_model.fit_generator(
train_gen,
steps_per_epoch=nbatches_train,
epochs=num_epochs,
validation_data=validation_gen,
validation_steps=nbatches_validation,
callbacks=[early_stopping, checkpoint])
# plot accuracy and loss for train and validation
if num_epochs > 1:
plots.plot_accuracy(results_dir, 'accuracy_epochs', history)
plots.plot_loss(results_dir, 'loss_epochs', history)
# load best checkpoint
checkpoint_filename = 'checkpoint.hdf5'
checkpoint_dir = os.path.join('checkpoints', str(bin_size))
checkpoint_path = os.path.join(checkpoint_dir, checkpoint_filename)
pretrained_model = load_model(checkpoint_path,
custom_objects={
'root_mean_squared_error':
root_mean_squared_error,
'mean_absolute_bin_error':
mean_absolute_bin_error
})
# predict on test set
test_gen = custom_generator(df_test, bin_size, False, (224, 224), 1)
optimizer.step()
acc_train = get_acc(output, adj_label)
hidden_emb = model.mu.data.numpy()
roc_curr, ap_curr = get_roc_score(hidden_emb, adj_orig, val_edges,
val_edges_false)
print('Epoch: {:04d}'.format(epoch + 1),
'loss_train: {:.4f}'.format(curr_loss),
'acc_train: {:.4f}'.format(acc_train), "val_ap=",
"{:.5f}".format(ap_curr), "val_roc=", "{:.5f}".format(roc_curr),
'time: {:.4f}s'.format(time.time() - t))
for epoch in range(args.epochs):
train(epoch)
hidden_emb = model.mu.data.numpy()
roc_score, ap_score = get_roc_score(hidden_emb, adj_orig, test_edges,
test_edges_false)
print('Test ROC score: ' + str(roc_score))
print('Test AP score: ' + str(ap_score))
auc_roc(hidden_emb, adj_orig, test_edges, test_edges_false, roc_score,
"Ontology Homology")
torch.save(model.state_dict(), "models/gae_onto_hom")
plot_loss(indices, losses, "Epooch", "Loss", "Epooch vs loss")
def main():
'''
This is the CVAE model for trajectory prediction. It has:
three induvidual encoders for heatmap/scene, occupancy grid, and trajectry sequence
one CVAE encoder for latent z
one CVAE encoder for generating predictions
'''
timestr = time.strftime("%Y%m%d-%H%M%S")
dataname = 'HC'
print(dataname)
# Make all the necessary folders
mak_dir()
bg_image, data_dir, dirs, filename = get_data_repo(dataname)
# Get the hyperparameters
args = parse_args(dataname)
num_pred = args.num_pred
obs_seq = args.obs_seq - 1 ### minus one is for residual
pred_seq = args.pred_seq
neighSize = args.neighSize
gridRadius = args.gridRadius
gridAngle = args.gridAngle
train_mode = args.train_mode
# retrain_mode = args.retrain_mode
sceneType = args.sceneType
n_hidden = args.n_hidden
z_dim = args.z_dim
encoder_dim = args.encoder_dim
z_decoder_dim = args.z_decoder_dim
hidden_size = args.hidden_size
batch_size = args.batch_size
h_drop = args.h_drop
o_drop = args.o_drop
s_drop = args.s_drop
dropout = args.dropout
lr = args.lr
epochs = args.epochs
scale = args.scale
real_scale = args.real_scale
beta = args.beta
resi_scale = args.resi_scale
# Generate data
if args.data_process == True:
generate_data(dataname, leaveoneout=args.leaveoneout)
#################### MODEL CONSTRUCTION STARTS FROM HERE ####################
# Define the ResNet for scene
# Get the parsed last shape of the heatmap input
parse_dim = 1280 # The output dimension of MobileNet
# Please note that here you can also train the feacture using the CNN from scratch
# CONSTRUCT THREE LSTM ENCODERS TO ENCODE HEATMAP, OCCUPANCY GRID, AND TRAJECTORY INFORMATION IN PARALLEL
# Construct the heatmap scene model
# Sence model for the observed data
s_obs_in = Input(shape=(obs_seq, parse_dim), name='s_obs_in')
s_obs_out = LSTM(hidden_size,
return_sequences=False,
stateful=False,
dropout=h_drop,
name='h_obs_out')(s_obs_in)
s_obs_Model = Model(s_obs_in, s_obs_out)
s_obs_Model.summary()
# scene model for the conditioned data
s_pred_in = Input(shape=(pred_seq, parse_dim), name='s_pred_in')
s_pred_out = LSTM(hidden_size,
return_sequences=False,
stateful=False,
dropout=h_drop,
name='s_pred_out')(s_pred_in)
# Construct the occupancy grid model
# occupancy grid model for the observed data
o_obs_in = Input(shape=(obs_seq,
int(neighSize / gridRadius * 360 / gridAngle)),
name='o_obs_in')
o_obs_out = LSTM(hidden_size,
return_sequences=False,
stateful=False,
dropout=o_drop,
name='o_obs_out')(o_obs_in)
o_obs_Model = Model(o_obs_in, o_obs_out)
o_obs_Model.summary()
# occupancy grid model for the conditioned data
o_pred_in = Input(shape=(pred_seq,
int(neighSize / gridRadius * 360 / gridAngle)),
name='o_pred_in')
o_pred_out = LSTM(hidden_size,
return_sequences=False,
stateful=False,
dropout=o_drop,
name='o_pred_out')(o_pred_in)
o_pred_Model = Model(o_pred_in, o_pred_out)
o_pred_Model.summary()
# Construct the sequence model
# sequence model for the observed data
# x_state
x = Input(shape=(obs_seq, 5),
name='x') # including the 3-dimensions for one-hot encoding
x_conv1d = Conv1D(n_hidden // 16,
kernel_size=3,
strides=1,
padding='same',
name='x_conv1d')(x)
# Do I need to have a activation function?
x_dense = Dense(n_hidden // 8, activation='relu', name='x_dense')(x_conv1d)
x_state = LSTM(n_hidden // 8,
return_sequences=False,
stateful=False,
dropout=s_drop,
name='x_state')(x_dense) # (1, 64)
# encoded x
x_endoced = concatenate([x_state, s_obs_out, o_obs_out], name='x_endoced')
x_encoded_dense = Dense(encoder_dim,
activation='relu',
name='x_encoded_dense')(x_endoced)
# sequence model for the conditioned model
# y_state
y = Input(shape=(pred_seq, 2), name='y')
y_conv1d = Conv1D(n_hidden // 16,
kernel_size=3,
strides=1,
padding='same',
name='y_conv1d')(y)
y_dense = Dense(n_hidden // 8, activation='relu', name='y_dense')(y_conv1d)
y_state = LSTM(n_hidden // 8,
return_sequences=False,
stateful=False,
dropout=s_drop,
name='y_state')(y_dense) # (1, 64)
# encoded y
y_encoded = concatenate([y_state, s_pred_out, o_pred_out],
name='y_encoded')
y_encoded_dense = Dense(encoder_dim,
activation='relu',
name='y_encoded_dense')(y_encoded)
# CONSTRUCT THE CVAE ENCODER BY FEEDING THE CONCATENATED ENCODED HEATMAP, OCCUPANCY GRID, AND TRAJECTORY INFORMATION
# the concatenated input
inputs = concatenate([x_encoded_dense, y_encoded_dense],
name='inputs') # (1, 256)
xy_encoded_d1 = Dense(n_hidden, activation='relu',
name='xy_encoded_d1')(inputs) # (1, 512)
xy_encoded_d2 = Dense(n_hidden // 2,
activation='relu',
name='xy_encoded_d2')(xy_encoded_d1) # (1, 256)
mu = Dense(z_dim, activation='linear', name='mu')(xy_encoded_d2) # 2
log_var = Dense(z_dim, activation='linear',
name='log_var')(xy_encoded_d2) # 2
# THE REPARAMETERIZATION TRICK FOR THE LATENT VARIABLE z
# sampling function
def sampling(params):
mu, log_var = params
eps = K.random_normal(shape=(K.shape(mu)[0], z_dim),
mean=0.,
stddev=1.0)
return mu + K.exp(log_var / 2.) * eps
# sampling z
z = Lambda(sampling, output_shape=(z_dim, ), name='z')([mu, log_var])
# concatenate the z and x_encoded_dense
z_cond = concatenate([z, x_encoded_dense], name='z_cond')
# CONSTRUCT THE CVAE DECODER
z_decoder1 = Dense(n_hidden // 2, activation='relu', name='z_decoder1')
z_decoder2 = RepeatVector(pred_seq, name='z_decoder2')
z_decoder3 = LSTM(z_decoder_dim,
return_sequences=True,
stateful=False,
dropout=dropout,
name='z_decoder3')
z_decoder4 = Activation('tanh', name='z_decoder4')
z_decoder5 = Dropout(dropout, name='z_decoder5')
y_decoder = TimeDistributed(Dense(2), name='y_decoder') # (12, 2)
# Instantiate the decoder by feeding the concatenated z and x_encoded_dense
z_d1 = z_decoder1(z_cond)
z_d2 = z_decoder2(z_d1)
z_d3 = z_decoder3(z_d2)
z_d4 = z_decoder4(z_d3)
z_d5 = z_decoder5(z_d4)
y_prime = y_decoder(z_d5)
# CONSTRUCT THE LOSS FUNCTION FOR THE CVAE MODEL
### Update the utility of the loss function
def vae_loss(y, y_prime):
'''
This is the customized loss function
'''
reconstruction_loss = K.mean(mse(y, y_prime) * pred_seq)
kl_loss = 0.5 * K.sum(K.square(mu) + K.exp(log_var) - log_var - 1,
axis=-1)
cvae_loss = K.mean(reconstruction_loss * beta + kl_loss * (1 - beta))
return cvae_loss
# BUILD THE CVAE MODEL
cvae = Model([s_obs_in, s_pred_in, o_obs_in, o_pred_in, x, y], [y_prime])
opt = optimizers.Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
decay=0.0,
amsgrad=False)
cvae.compile(optimizer=opt, loss=vae_loss)
cvae.summary()
# CHECK POINT AND SAVE THE BEST MODEL
filepath = "../models/cvae_mixed_%s_%0.f_%s.hdf5" % (dataname, epochs,
timestr)
## ToDo, Eraly stop
earlystop = EarlyStopping(monitor='val_loss',
mode='min',
verbose=1,
patience=50)
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=0,
save_best_only=True,
mode='min')
callbacks_list = [earlystop, checkpoint]
# Load the data
# No mattter train_mode or retrain mode, the validation and test data are the same and they are loaded once the model is built.
# For the 7030 tarining and test seting on the same set, the name of training (30%) and validation (70%) are swapped
print('Load the validation data ...')
V_ = np.load('../processed/%s_validation_data_grid.npz' % dataname)
val_obs, val_pred, val_raw, val_obs_og, val_pred_og, val_obs_hmaps, val_pred_hmaps = V_[
'obs'], V_['pred'], V_['raw'], V_['obs_og'], V_['pred_og'], V_[
'obs_hmaps'], V_['pred_hmaps']
# Shift one time step for occupancy grid and scene attention
val_obs_og = val_obs_og[:, 1:, :]
val_obs_hmaps = val_obs_hmaps[:, 1:, ...]
# Get the residual of the trajectories
val_traj = np.concatenate((val_obs[:, :, 2:4], val_pred[:, :, 2:4]),
axis=1)
val_traj_r = val_traj[:, 1:, :] - val_traj[:, :-1, :]
val_traj_r = val_traj_r * resi_scale
val_obs_r, val_pred_r = val_traj_r[:, :obs_seq, :], val_traj_r[:,
obs_seq:, :]
## Get User type one-hot encoding
val_obs_type = get_type(val_obs, 3, isObservation=True)
val_obs_r = np.concatenate((val_obs_r, val_obs_type), axis=-1)
# Double check the data shape
# Here is the validation realted data
print('the shape of val_obs', val_obs_r.shape)
print('the shape of val_pred', val_pred_r.shape)
print('the number of road users in val_raw', len(np.unique(val_raw[:, 0])))
print('the shape of val_obs_og', val_obs_og.shape)
print('the shape of val_pred_og', val_pred_og.shape)
print('the shape of val_obs_hmaps', val_obs_hmaps.shape)
print('the shape of val_pred_hmaps', val_pred_hmaps.shape)
# Construct the heatmap/scene inputs for the model
# Please note that during training and validation, in order to calculate latent z, y (ground truth prediction) is visiable
# y (ground truth prediction) is not available in testing
val_obs_hpinput = mobilenet(val_obs_hmaps, obs_seq)
val_pred_hpinput = mobilenet(val_pred_hmaps, pred_seq)
print('\nLoad the test data ...')
# =============================================================================
# test = np.load('../processed/%s_validation_data_grid.npz'%dataname)
# test_obs, test_pred, test_raw, test_obs_og, test_obs_hmaps = test['obs'], test['pred'], test['raw'], test['obs_og'], test['obs_hmaps']
# # Shift one time step for occupancy grid and scene attention
# test_obs_og = test_obs_og[:, 1:, :]
# test_obs_hmaps = test_obs_hmaps[:, 1:, ...]
# # Get the residual of the trajectories
# test_traj = np.concatenate((test_obs[:, :, 2:4], test_pred[:, :, 2:4]), axis=1)
# test_traj_r = test_traj[:, 1:, :] - test_traj[:, :-1, :]
# test_traj_r = test_traj_r*resi_scale
# test_obs_r = test_traj_r[:, :obs_seq, :]
# ## Get User type one-hot encoding
# test_obs_type = get_type(test_obs, 3, isObservation=True)
# test_obs_r = np.concatenate((test_obs_r, test_obs_type), axis=-1)
# =============================================================================
test_raw = val_raw
test_obs = val_obs
test_pred = val_pred
test_obs_r = val_obs_r
# test_pred_r = val_pred_r
test_obs_og = val_obs_og
# test_pred_og = val_pred_og
test_obs_hmaps = val_obs_hmaps
# test_pred_hmaps = val_pred_hmaps
test_obs_hpinput = val_obs_hpinput
# test_pred_hpinput = val_pred_hpinput
# Double check the data shape
# Here is the testing related data
print('the shape of test_obs', test_obs_r.shape)
print('the number of road users in test_raw', len(np.unique(test_raw[:,
0])))
print('the shape of test_obs_og', test_obs_og.shape)
print('the shape of test_obs_hmaps', test_obs_hmaps.shape)
# Construct the heatmap/scene inputs for the model
# Please note that during training and validation, in order to calculate latent z, y (ground truth prediction) is visiable
# y (ground truth prediction) is not available in testing
test_obs_hpinput = mobilenet(test_obs_hmaps, obs_seq)
#################### START TRAINING THE CVAE MODEL ####################
if train_mode:
print("\nload the training data")
T_ = np.load('../processed/%s_training_data_grid.npz' % dataname)
train_obs, train_pred, train_obs_og, train_pred_og, train_obs_hmaps, train_pred_hmaps = T_[
'obs'], T_['pred'], T_['obs_og'], T_['pred_og'], T_[
'obs_hmaps'], T_['pred_hmaps']
print('You are using the scene context from %s' % sceneType)
print('Data loaded!')
# Shift one time step for occupancy grid and scene attention
train_obs_og = train_obs_og[:, 1:, :]
train_obs_hmaps = train_obs_hmaps[:, 1:, ...]
# Get the residual of the trajectories
train_traj = np.concatenate(
(train_obs[:, :, 2:4], train_pred[:, :, 2:4]), axis=1)
train_traj_r = train_traj[:, 1:, :] - train_traj[:, :-1, :]
train_traj_r = train_traj_r * resi_scale
train_obs_r, train_pred_r = train_traj_r[:, :
obs_seq, :], train_traj_r[:,
obs_seq:, :]
## Get User type one-hot encoding
train_obs_type = get_type(train_obs, 3, isObservation=True)
train_obs_r = np.concatenate((train_obs_r, train_obs_type), axis=-1)
# Here is the training related data
print('Check the input data...')
print('the shape of train_obs', train_obs_r.shape)
print('the shape of train_pred', train_pred_r.shape)
#print('the number of road users in train_raw', len(np.unique(train_raw[:, 0])))
print('the shape of train_obs_og', train_obs_og.shape)
print('the shape of train_pred_og', train_pred_og.shape)
print('the shape of train_obs_hmaps', train_obs_hmaps.shape)
print('the shape of train_pred_hmaps', train_pred_hmaps.shape)
# Construct the heatmap/scene inputs for the model
# Please note that during training and validation, in order to calculate latent z, y (ground truth prediction) is visiable
# y (ground truth prediction) is not available in testing
train_obs_hpinput = mobilenet(train_obs_hmaps, obs_seq)
train_pred_hpinput = mobilenet(train_pred_hmaps, pred_seq)
print('Start training the CVAE model...')
history = cvae.fit(x=[
train_obs_hpinput, train_pred_hpinput, train_obs_og, train_pred_og,
train_obs_r, train_pred_r
],
y=train_pred_r,
shuffle=True,
epochs=epochs,
batch_size=batch_size,
verbose=1,
callbacks=callbacks_list,
validation_data=([
val_obs_hpinput, val_pred_hpinput, val_obs_og,
val_pred_og, val_obs_r, val_pred_r
], val_pred_r))
print('Training the CVAE model done!')
print('Plotting loss...')
plot_loss(history, scale, real_scale, timestr, dataname)
print('Plotting done!')
### Here load the best trained model
cvae.load_weights(filepath)
else:
print("\nLoad the trained model")
if args.sceneType == 'heatmap':
print("The scene context is heatmap")
trained_model = "../trained_model/....hdf5"
elif args.sceneType == "aerial_photograph":
print("The scene context is aerial photograph")
trained_model = "../trained_model/....hdf5"
elif args.sceneType == 'segmented_map':
print("The scene context is segmented map")
trained_model = "../trained_model/....hdf5"
cvae.load_weights(trained_model)
#################### START TESTING ####################
# NOTE THAT IN TESTING PHASE, ONLY x (observed trajectory) is available
# construct the encoder to get the x_encoded_dense, including heatmap, occupancy, and trajectory sequence information
print('Start testing...')
print('Construct the CVAE encoder')
x_encoder = Model([s_obs_in, o_obs_in, x], x_encoded_dense)
x_encoder.summary()
# get the x_encoded_dense as latent feature for prediction
x_latent = x_encoder.predict([test_obs_hpinput, test_obs_og, test_obs_r],
batch_size=batch_size)
# CONSTRUCT THE DECODER
print('Construct the CVAE decoder')
decoder_input = Input(shape=(z_dim + encoder_dim, ), name='decoder_input')
_z_d1 = z_decoder1(decoder_input)
_z_d2 = z_decoder2(_z_d1)
_z_d3 = z_decoder3(_z_d2)
_z_d4 = z_decoder4(_z_d3)
_z_d5 = z_decoder5(_z_d4)
_y_prime = y_decoder(_z_d5)
generator = Model(decoder_input, _y_prime)
generator.summary()
# Save the summary of the model
with open(
'../models/cvae_mixed_%s_model_summary_%s.txt' %
(dataname, timestr), 'w') as f:
with redirect_stdout(f):
# s_obs_Model.summary()
# o_obs_Model.summary()
# o_pred_Model.summary()
cvae.summary()
x_encoder.summary()
generator.summary()
# START PREDICTING USING THE x_encoded_dense AND SAMPLED LATENT VARIABLE z
print('Start predicting...')
### Change the residual back to positions
last_obs_test = test_obs[:, -1, 2:4]
predictions = []
for i, x_ in enumerate(x_latent):
last_pos = last_obs_test[i]
x_ = np.reshape(x_, [1, -1])
for i in range(num_pred):
# sampling z from a normal distribution
z_sample = np.random.rand(1, z_dim)
y_p = generator.predict(np.column_stack([z_sample, x_]))
y_p = y_p / resi_scale
y_p_ = np.concatenate(([last_pos], np.squeeze(y_p)), axis=0)
y_p_sum = np.cumsum(y_p_, axis=0)
predictions.append(y_p_sum[1:, :])
predictions = np.reshape(predictions, [-1, num_pred, pred_seq, 2])
indexed_predictions = get_index_prediction(test_obs, test_pred,
predictions, num_pred, pred_seq)
print('Predicting done!')
## this is the error for the sampling (aveage and minimum)
classfied_errors = get_classified_errors(test_pred, indexed_predictions,
scale / real_scale)
## Select the most likely prediction based on Gaussian rank
selected_preds = np.zeros((0, 5))
for pred in indexed_predictions:
select_index = gauss_rank(pred)
for m, traj in enumerate(pred):
if m == select_index:
selected_preds = np.vstack((selected_preds, traj))
selected_preds = selected_preds.reshape(indexed_predictions.shape[0], 1,
indexed_predictions.shape[2],
indexed_predictions.shape[3])
selected_errors = get_classified_errors(test_pred, selected_preds,
scale / real_scale)
np.savetxt('../results/cvae_mixed_%s_statistic_results_%.2f_%s.txt' %
(dataname, classfied_errors[0, 2], timestr),
classfied_errors,
delimiter=',')
np.savetxt(
'../selected_results/cvae_mixed_%s_statistic_results_%.2f_%s.txt' %
(dataname, selected_errors[0, 2], timestr),
selected_errors,
delimiter=',')
# Save the predictions
print('Start saving the predictions...')
np.save(
'../predictions/cvae_mixed_%s_test_obs_grid_%.2f_%s.npy' %
(dataname, classfied_errors[0, 2], timestr), test_obs)
np.save(
'../predictions/cvae_mixed_%s_test_gt_grid_%.2f_%s.npy' %
(dataname, classfied_errors[0, 2], timestr), test_pred)
np.save(
'../predictions/cvae_mixed_%s_test_predictions_grid_%.2f_%s.npy' %
(dataname, classfied_errors[0, 2], timestr), indexed_predictions)
print('Results saved...')
# Plotting the predicted trajectories
print('Start plotting the predictions...')
plot_dir = '../plots/cvae_mixed_%s_plot_%.2f_%s' % (
dataname, classfied_errors[0, 2], timestr)
if not os.path.exists(plot_dir):
os.mkdir(plot_dir)
print("Directory cvae_mixed_%s_plot_%.2f_%s" %
(dataname, classfied_errors[0, 2], timestr))
else:
print("Directory cvae_mixed_%s_plot_%.2f_%s" %
(dataname, classfied_errors[0, 2], timestr))
plot_scenarios(test_raw, test_obs, test_pred, indexed_predictions,
bg_image, scale, real_scale, plot_dir)
print('Plotting Done!')
# Save the model configuration
with open(
'../hyperparameters/cvae_mixed_%s_args_%.2f_%s.txt' %
(dataname, classfied_errors[0, 2], timestr), 'w') as f:
for arg in vars(args):
params = ('%s = %s' % (str(arg), str(getattr(args, arg))))
f.writelines(params)
f.writelines('\n')
weights = {'o_P': advantage, 'o_V': np.ones(len(advantage))}
#backpropagation
history = model.fit(episode_state, [episode_output, episode_r],
epochs=EPISODE + 1,
batch_size=len(episode_output),
callbacks=callbacks_list,
sample_weight=weights,
initial_epoch=EPISODE)
episode_r = []
episode_output = []
episode_state = np.zeros((0, IMAGE_ROWS, IMAGE_COLS, IMAGE_CHANNELS))
episode_critic = []
f = open("rewards/rewards@" + timestr + ".txt", "a")
f.write("Update: " + str(EPISODE) + ", Reward_mean: " + str(e_mean) +
", Loss: " + str(history.history['loss']) + "\n")
f.close()
if EPISODE > 50: # making sure the graph does not look too bad
plot_reward(e_mean)
plot_loss(history.history['loss'])
else:
plot_reward(0)
plot_loss([1])
if EPISODE % SAVE_FREQUENCY == 0:
model.save("saved_models/" + timestr + "/model_updates" + str(EPISODE))
EPISODE += 1
start = time.time()
print("========== TRAIN ============")
epochs = 0 # TODO: update according to the last model if IMPORT = True
for i in range(30):
rnn.train_across_buildings(train_mainslist,
train_meterlist,
val_mainslist,
val_meterlist,
epochs=10,
sample_period=sample_period)
epochs += 10
rnn.export_model(
os.path.join(results_dir,
"UKDALE-RNN-{}-{}epochs.h5".format(meter_key, epochs)))
plot_loss(train_logfile, val_logfile, results_dir)
print("CHECKPOINT {}".format(epochs))
end = time.time()
print("Train =", end - start, "seconds.")
headline = "========== DISAGGREGATE ============"
with open(results_file, "a") as text_file:
text_file.write(headline + '\n')
print(headline)
# find best model (min validation loss)
validation = pd.read_csv(val_logfile)
epochs = np.array(validation.as_matrix()[:, 0], dtype='int')
loss = np.array(validation.as_matrix()[:, 1], dtype='float32')
argmin = np.argmin(loss)
best_epoch = epochs[argmin] + 1
if iteration % 100 == 0:
counter.append(iteration)
loss_history_step.append(loss.item())
iteration += 1
fnet.eval()
train_acc.append(get_accuracy(test, 0.5, fnet))
loss_history.append(train_loss)
loss_history_d.append(train_d_loss)
loss_history_g.append(train_g_loss)
print("Epoch number {}\n Current loss {}\n".format(epoch, train_loss))
# Plot
plot_loss(loss_history, 'Loss')
plot_loss(loss_history_d, 'Training Discriminative Loss')
plot_loss(loss_history_g, 'Training Generative Loss')
plot_loss(train_acc, 'Accuracy')
# Test the model
fnet.eval()
positive = []
negative = []
tp = 0
tn = 0
fp = 0
fn = 0
threshold = 0.5
validation_data=validation_gen,
validation_steps=nbatches_validation,
callbacks=[early_stopping])
else:
history = pretrained_model.fit_generator(
train_gen,
steps_per_epoch=nbatches_train,
epochs=num_epochs,
validation_data=validation_gen,
validation_steps=nbatches_validation,
callbacks=[early_stopping])
# plot accuracy and loss for train and validation
if num_epochs > 1:
plots.plot_accuracy(figures_dir, 'accuracy_epochs_resnet50', history)
plots.plot_loss(figures_dir, 'loss_epochs_resnet50', history)
# load model
checkpoint_filename = 'resnet50_best.hdf5'
checkpoint_dir = os.path.join('checkpoints', str(bin_size))
checkpoint_path = os.path.join(checkpoint_dir, checkpoint_filename)
pretrained_model = load_model(
checkpoint_path,
custom_objects={'mean_absolute_bin_error': mean_absolute_bin_error})
# evaluate on test set
test_gen = custom_generator(df_test, label_encoder, bin_size, False,
(224, 224), 1)
test_results = pretrained_model.evaluate_generator(test_gen,
steps=nbatches_test)
test_loss = test_results[0]