def __build_model_and_summary(self):
# split the tensors
with tf.variable_scope("tower_split"), tf.device("/cpu:0"):
# splitted tensors
ts_inputs = tf.split(self.t_inputs, self.num_gpu, 1)
ts_lstm_initial_state = tf.split(self.t_lstm_initial_state, self.num_gpu, 2)
ts_initial_poses = tf.split(self.t_initial_poses, self.num_gpu, 0)
ts_imu_data = tf.split(self.t_imu_data, self.num_gpu, 1)
ts_ekf_initial_state = tf.split(self.t_ekf_initial_state, self.num_gpu, 0)
ts_ekf_initial_covar = tf.split(self.t_ekf_initial_covariance, self.num_gpu, 0)
ts_se3_labels = tf.split(self.t_se3_labels, self.num_gpu, 1)
ts_fc_labels = tf.split(self.t_fc_labels, self.num_gpu, 1)
# list to store results
ts_ekf_states = []
ts_ekf_covar_states = []
ts_lstm_states = []
losses_keys = ["se3_loss", "se3_xyz_loss", "se3_quat_loss",
"fc_loss", "fc_xyz_loss", "fc_ypr_loss", "x_loss", "y_loss", "z_loss",
"total_loss"]
ts_losses_dict = dict(zip(losses_keys, [[] for i in range(len(losses_keys))]))
for i in range(0, self.num_gpu):
device_setter = tf.train.replica_device_setter(ps_tasks=1,
ps_device='/job:localhost/replica:0/task:0/device:CPU:0',
worker_device='/job:localhost/replica:0/task:0/device:GPU:%d' % i)
with tf.name_scope("tower_%d" % i), tf.device(device_setter):
tools.printf("Building model...")
fc_outputs, fc_covar, se3_outputs, lstm_states, ekf_states, ekf_covar_states = \
model.build_seq_model(self.cfg, ts_inputs[i], ts_lstm_initial_state[i], ts_initial_poses[i],
ts_imu_data[i], ts_ekf_initial_state[i], ts_ekf_initial_covar[i],
self.t_is_training, get_activations=True,
use_initializer=self.t_use_initializer,
use_ekf=self.cfg.use_ekf)
# this returns lstm states as a tuple, we need to stack them
lstm_states = tf.stack(lstm_states, 0)
ts_lstm_states.append(lstm_states)
ts_ekf_states.append(ekf_states)
ts_ekf_covar_states.append(ekf_covar_states)
with tf.variable_scope("loss"):
se3_loss, se3_xyz_loss, se3_quat_loss \
= losses.se3_losses(se3_outputs, ts_se3_labels[i], self.cfg.k_se3)
fc_loss, fc_xyz_loss, fc_ypr_loss, x_loss, y_loss, z_loss \
= losses.fc_losses(fc_outputs, fc_covar, ts_fc_labels[i], self.cfg.k_fc)
total_loss = (1 - self.t_alpha) * se3_loss + self.t_alpha * fc_loss
for k, v in ts_losses_dict.items():
v.append(locals()[k])
tf.get_variable_scope().reuse_variables()
with tf.variable_scope("tower_join"), tf.device("/cpu:0"):
# join the lstm states
self.t_lstm_states = tf.concat(ts_lstm_states, 2)
for k, v in ts_losses_dict.items():
ts_losses_dict[k] = tf.reduce_mean(v)
self.t_ekf_states = tf.concat(ts_ekf_states, 0)
self.t_ekf_covar_states = tf.concat(ts_ekf_covar_states, 0)
self.t_total_loss = ts_losses_dict["total_loss"]
self.t_se3_loss = ts_losses_dict["se3_loss"]
tools.printf("Building optimizer...")
with tf.variable_scope("optimizer", reuse=tf.AUTO_REUSE):
if self.cfg.use_init and self.cfg.only_train_init:
train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "initializer_layer")
elif self.cfg.train_noise_covariance and self.cfg.static_nn:
train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "imu_noise_params")
else:
train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
self.op_trainer = tf.train.AdamOptimizer(learning_rate=self.t_lr). \
minimize(self.t_total_loss, colocate_gradients_with_ops=True, var_list=train_vars)
# tensorboard summaries
tools.printf("Building tensorboard summaries...")
with tf.device("/cpu:0"):
self.t_sequence_id = tf.placeholder(dtype=tf.uint8, shape=[])
self.t_epoch = tf.placeholder(dtype=tf.int32, shape=[])
tf.summary.scalar("total_loss", ts_losses_dict["total_loss"])
tf.summary.scalar("fc_loss", ts_losses_dict["fc_loss"])
tf.summary.scalar("se3_loss", ts_losses_dict["se3_loss"])
tf.summary.scalar("fc_xyz_loss", ts_losses_dict["fc_xyz_loss"])
tf.summary.scalar("fc_ypr_loss", ts_losses_dict["fc_ypr_loss"])
tf.summary.scalar("se3_xyz_loss", ts_losses_dict["se3_xyz_loss"])
tf.summary.scalar("se3_quat_loss", ts_losses_dict["se3_quat_loss"])
tf.summary.scalar("x_loss", ts_losses_dict["x_loss"])
tf.summary.scalar("y_loss", ts_losses_dict["y_loss"])
tf.summary.scalar("z_loss", ts_losses_dict["z_loss"])
tf.summary.scalar("alpha", self.t_alpha)
tf.summary.scalar("lr", self.t_lr)
tf.summary.scalar("sequence_id", self.t_sequence_id)
tf.summary.scalar("epoch", self.t_epoch)
self.op_train_merged_summary = tf.summary.merge_all()
activations = tf.get_collection(tf.GraphKeys.ACTIVATIONS)
initial_layer = tf.summary.image("1st layer activations", tf.expand_dims(activations[0][:, 0, :, :], -1))
final_layer = tf.summary.image("Last layer activations", tf.expand_dims(activations[1][:, 0, :, :], -1))
self.op_train_image_summary = tf.summary.merge([initial_layer, final_layer])
val_loss_sum = tf.summary.scalar("val_total_loss", ts_losses_dict["total_loss"])
val_fc_sum = tf.summary.scalar("val_fc_losses", ts_losses_dict["fc_loss"])
val_se3_sum = tf.summary.scalar("val_se3_losses", ts_losses_dict["se3_loss"])
val_fc_xyz_loss = tf.summary.scalar("val_fc_xyz_loss", ts_losses_dict["fc_xyz_loss"])
val_fc_ypr_loss = tf.summary.scalar("val_fc_ypr_loss", ts_losses_dict["fc_ypr_loss"])
val_se3_xyz_loss = tf.summary.scalar("val_se3_xyz_loss", ts_losses_dict["se3_xyz_loss"])
val_se3_quat_loss = tf.summary.scalar("val_se3_quat_loss", ts_losses_dict["se3_quat_loss"])
val_x_sum = tf.summary.scalar("val_x_loss", ts_losses_dict["x_loss"])
val_y_sum = tf.summary.scalar("val_y_loss", ts_losses_dict["y_loss"])
val_z_sum = tf.summary.scalar("val_z_loss", ts_losses_dict["z_loss"])
self.op_val_merged_summary = tf.summary.merge(
[val_loss_sum, val_fc_sum, val_se3_sum, val_fc_xyz_loss, val_fc_ypr_loss, val_se3_xyz_loss,
val_se3_quat_loss, val_x_sum, val_y_sum, val_z_sum])
cfg.use_init = False
cfg_si.timesteps = 1
cfg_si.sequence_stride = 1
cfg_si.batch_size = 1
cfg_si.bidir_aug = False
# cfg_si.use_init = what ever the original setting was
tools.printf("Building eval model....")
inputs, lstm_initial_state, initial_poses, imu_data, ekf_initial_state, ekf_initial_covariance, _, _, dt \
= model.seq_model_inputs(cfg)
fc_outputs, fc_covar, se3_outputs, lstm_states, ekf_out_states, ekf_out_covar, _, _, _ = \
model.build_seq_model(cfg, inputs, lstm_initial_state, initial_poses, imu_data, ekf_initial_state,
ekf_initial_covariance,
dt,
tf.constant(False, dtype=tf.bool), # is training
False, # get_activation
tf.constant(False, dtype=tf.bool), # use initializer
cfg.use_ekf) # use ekf
if cfg_si.use_init:
tools.printf("Building eval model for initial LSTM states...")
inputs_si, _, initial_poses_si, imu_data_si, _, _, _, _ = model.seq_model_inputs(cfg_si)
_, _, _, _, _, _, feed_lstm_initial_states, feed_ekf_inital_states, feed_initial_covariance = \
model.build_seq_model(cfg_si, inputs_si,
tf.constant(np.zeros([2, cfg_si.lstm_layers, cfg_si.batch_size, cfg_si.lstm_size]),
dtype=tf.float32),
initial_poses_si,
imu_data_si,
np.zeros([cfg_si.batch_size, 17], dtype=np.float32),
0.01 * np.repeat(np.expand_dims(np.identity(17, dtype=np.float32), axis=0),
alpha_set = 0.5
tensorboard_meta = False
# =================== MODEL + LOSSES + Optimizer ========================
tools.printf("Building losses...")
with tf.device("/cpu:0"):
alpha = tf.placeholder(tf.float32, name="alpha",
shape=[]) # between 0 and 1, larger favors fc loss
with tf.device(
tf.train.replica_device_setter(
ps_tasks=1,
ps_device='/job:localhost/replica:0/task:0/device:GPU:0',
worker_device='/job:localhost/replica:0/task:0/device:GPU:0')):
inputs, lstm_initial_state, initial_poses, is_training, fc_outputs, se3_outputs, lstm_states = model.build_seq_model(
cfg, True)
se3_labels, fc_labels = simple_model.model_labels(cfg)
with tf.variable_scope("Losses"):
se3_losses, se3_xyz_losses, se3_quat_losses = losses.se3_losses(
se3_outputs, se3_labels, cfg.k_se3)
fc_losses, fc_xyz_losses, fc_ypr_losses, \
x_loss, y_loss, z_loss = losses.pair_train_fc_losses(fc_outputs, fc_labels, cfg.k_fc)
total_losses = (1 - alpha) * se3_losses + alpha * fc_losses
tools.printf("Building optimizer...")
with tf.variable_scope("Optimizer"):
with tf.device("/gpu:0"):
# dynamic learning rates
lr = tf.placeholder(tf.float32, name="se3_lr", shape=[])
trainer = tf.train.AdamOptimizer(learning_rate=lr).minimize(
dir_name = "trajectory_results"
kitti_seqs = ["00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "10"]
#kitti_seqs = ["01"]
# if kitti_seq in ["11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21"]:
# save_ground_truth = False
# else:
# save_ground_truth = True
save_ground_truth = True
cfg = config.SeqEvalLidarConfig
tools.printf("Building eval model....")
inputs, lstm_initial_state, initial_poses, \
is_training, fc_outputs, se3_outputs, lstm_states = model.build_seq_model(cfg)
for kitti_seq in kitti_seqs:
tools.printf("Loading training data...")
train_data_gen = data.StatefulRollerDataGen(cfg, config.dataset_path,
[kitti_seq])
results_dir_path = os.path.join(config.save_path, dir_name)
if not os.path.exists(results_dir_path):
os.makedirs(results_dir_path)
# ==== Read Model Checkpoints =====
restore_model_file = "/home/cs4li/Dev/end_to_end_visual_odometry/results/train_seq_20180418-16-37-02/best_val/model_best_val_checkpoint-143"
variable_to_load = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
"^(cnn_layer|rnn_layer|fc_layer).*")