history_teacher = LossAccHistory()
training = KDModel.NormalTraining(teacher_model)
teacher_model.summary()
# plot_model(teacher_model, show_shapes=True, to_file='teacher_model.png')
for epoch in range(1, EPOCHS_T + 1):
loss_metric = Mean()
loss_metric_val = Mean()
acc_metric = Mean()
acc_metric_val = Mean()
acc = CategoricalAccuracy()
acc_val = CategoricalAccuracy()
# 各バッチごとに学習
for (x_train_, y_train_), (x_val_, y_val_) in ds:
loss_value, grads = training.grad(x_train_, y_train_)
optimizer.apply_gradients(zip(grads, teacher_model.trainable_weights))
loss_value_test = training.loss(x_val_, y_val_)
probs = tf.nn.softmax(teacher_model(x_train_))
probs_val = tf.nn.softmax(teacher_model(x_val_))
loss_metric(loss_value)
acc_metric(acc(y_train_, probs))
loss_metric_val(loss_value_test)
acc_metric_val(acc_val(y_val_, probs_val))
# 学習進捗の表示
print(
'Epoch {}/{}: Loss: {:.3f}, Accuracy: {:.3%}, Validation Loss: {:.3f}, Validation Accuracy: {:.3%}'
.format(epoch, EPOCHS_T,
loss_metric.result().numpy(),
acc_metric.result().numpy(),
class Trainer:
def __init__(self, config):
self.predict_only = config['Trainer']['predict_only']
self.nbr_samples_per_update = config['Trainer']['nbr_samples_per_update']
self.optimizer = Adamax(lr=1e-3)
self.buffered_samples = []
def run(self, sample, network):
loss = -tf.constant(float('Inf'), dtype=tf.float32)
recon_loss = -tf.constant(float('Inf'), dtype=tf.float32)
if sample is not None:
if self.predict_only:
loss = sample.predict(network)
return loss, recon_loss
else:
self.buffered_samples.append(sample)
if len(self.buffered_samples) >= self.nbr_samples_per_update:
trainable_variables = network.getTrainableVariables()
for sample in self.buffered_samples:
alpha = sample.getAlpha()
trainable_variables = [
*trainable_variables, *alpha]
z = sample.getZ()
trainable_variables = [
*trainable_variables, *z]
with tf.GradientTape(watch_accessed_variables=False) as tape:
tape.watch(trainable_variables)
loss = tf.constant(0.0, dtype=tf.float32)
recon_loss = tf.constant(0.0, dtype=tf.float32)
for sample in self.buffered_samples:
recon_loss_i, loss_i = sample.train(
network)
loss += loss_i
recon_loss += recon_loss_i
gradients = tape.gradient(
loss, trainable_variables)
self.optimizer.apply_gradients(
zip(gradients, trainable_variables))
loss /= len(self.buffered_samples)
recon_loss /= len(self.buffered_samples)
self.buffered_samples.clear()
return loss, recon_loss
class DepthOptimizer:
def __init__(self, config):
self.optimizer = Adamax(lr=1e-3)
self.g = Graphics3()
def get_graphics(self):
return self.g
def updateLearningRate(self, lr):
self.optimizer.learning_rate.assign(lr)
def getLearningRate(self):
return self.optimizer.learning_rate
def getCheckPointVariables(self):
return {"depth_optimizer": self.optimizer}
def predict(self, I, D, mask, z, alpha, calib, network):
I_batch = tf.stack(I)
D_batch = tf.stack(D)
mask_batch = tf.stack(mask)
calib_batch = tf.stack(calib)
R = self.g.normalized_points(I_batch, calib_batch)
IR = tf.concat((I_batch, R), axis=-1)
q, mu, logvar = network.encode(IR)
trainable_variables = z
rel_error = tf.constant(0.0, dtype=tf.float32)
for i in range(200):
with tf.GradientTape(watch_accessed_variables=False) as tape:
tape.watch(trainable_variables)
z_batch = tf.stack(z)
alpha_batch = tf.stack(alpha)
loss_next = self.predict_update(D_batch, mask_batch, q,
z_batch, alpha_batch, network)
loss_next += self.g.log_normal_loss(z_batch, mu, logvar)
gradients = tape.gradient(loss_next, trainable_variables)
self.optimizer.apply_gradients(
zip(gradients, trainable_variables))
if i > 0:
rel_error = 0.8*(tf.abs(loss_prev-loss_next) /
loss_next) + 0.2 * rel_error
if i > 0 and rel_error < 1e-3:
break
loss_prev = loss_next
return loss_next
@tf.function(experimental_compile=False)
def predict_update(self, D_batch, mask_batch, q, z_batch, alpha_batch, network):
shape = tf.shape(D_batch)
D_p = network.decode(q, z_batch)
loss_val = self.loss(D_batch, mask_batch, D_p, alpha_batch)
return loss_val
def train(self, I, D, mask, z, alpha, calib, network):
I_batch = tf.stack(I)
D_batch = tf.stack(D)
mask_batch = tf.stack(mask)
alpha_batch = tf.stack(alpha)
z_batch = tf.stack(z)
recon_loss, loss_val = self.train_loss(
I_batch, D_batch, mask_batch, z_batch, alpha_batch,
calib, network)
return recon_loss, loss_val
@tf.function(experimental_compile=False)
def train_loss(self, I_batch, D_batch, mask_batch, z_batch, alpha_batch, calib, network):
shape = tf.shape(I_batch)
calib_batch = tf.stack(calib)
R = self.g.normalized_points(I_batch, calib_batch)
IR = tf.concat((I_batch, R), axis=-1)
P, mu, logvar = network(IR, z_batch)
D_p = network.mapToDepth(alpha_batch, P)
recon_loss = self.loss(D_batch, mask_batch, D_p, alpha_batch)
reg_loss = network.sum_losses()
loss_val = recon_loss + reg_loss
loss_val += self.g.log_normal_loss(z_batch, mu, logvar)
return recon_loss, loss_val
def mapToProximal(self, A, Dinv):
shape = tf.shape(Dinv)
A = tf.reshape(A, shape=[shape[0], 1, 1, 1])
return tf.math.divide_no_nan(A*tf.ones_like(Dinv), Dinv + A*tf.ones_like(Dinv))
@tf.function(experimental_compile=False)
def loss(self, y_true, mask, y_pred, alpha_batch):
return tf.reduce_sum(tf.boolean_mask(tf.math.square(y_true-y_pred), mask))/0.01
@tf.function(experimental_compile=False)
def loss2(self, y_true, mask, y_pred, alpha_batch):
Diff = y_true-y_pred
shape = tf.shape(Diff)
Diff = Diff/tf.reshape(alpha_batch, shape=[shape[0], 1, 1, 1])
loss = tf.boolean_mask(self.g.Huber(Diff, 0.1), mask)
return tf.reduce_sum(loss)/0.01
class PhotometricOptimizer2:
def __init__(self, config):
self.max_iterations = config['PhotometricOptimizer']['max_iterations']
self.termination_crit = config['PhotometricOptimizer'][
'termination_crit']
self.image_height = config['dataset']['image_height']
self.image_width = config['dataset']['image_width']
self.g = Graphics3()
self.optimizer = Adamax(lr=1e-3)
self.timer = Timer(config)
self.angle_th = np.cos(
(config['PhotometricOptimizer']['angle_th'] / 180.0) * np.pi)
self.angle_th = tf.constant(self.angle_th, dtype=tf.float32)
def updateLearningRate(self, lr):
self.optimizer.learning_rate.assign(lr)
def getLearningRate(self):
return self.optimizer.learning_rate
def getCheckPointVariables(self):
return {"photometric_optimizer": self.optimizer}
def predict(self, I, z, alpha, T, Tinv, calib, network):
t1 = time.perf_counter() * 1000
self.timer.start()
I_batch = tf.stack(I)
T_batch = tf.stack(T)
Tinv_batch = tf.stack(Tinv)
calib_batch = tf.stack(calib)
alpha_batch = tf.stack(alpha)
self.timer.log("stack")
R = self.g.normalized_points(I_batch, calib_batch)
IR = tf.concat((I_batch, R), axis=-1)
q, mu, logvar = network.encode(IR)
self.timer.log("setup")
s = tf.constant(1.0, dtype=tf.float32)
count = tf.constant(0.0, dtype=tf.float32)
loss_val = tf.constant(-10.0, dtype=tf.float32)
mu_unstacked = tf.unstack(mu)
sig_unstacked = tf.unstack(tf.math.exp(logvar))
trainable_variables = [*z, *alpha]
for i, e in enumerate(mu_unstacked):
if tf.reduce_sum(tf.abs(z[i])) == 0.0:
z[i].assign(mu_unstacked[i])
for i in range(self.max_iterations):
z_batch = tf.stack(z)
self.timer.log("gradient")
gradients, loss_val, s, count, stop = self.calculate_gradients(
s, T_batch, Tinv_batch, calib_batch, I_batch, alpha_batch,
z_batch, q, mu, logvar, network, loss_val, count)
self.timer.log("update")
g = [tf.unstack(gradient) for gradient in gradients]
flat_list = [item for sublist in g for item in sublist]
self.optimizer.apply_gradients(zip(flat_list, trainable_variables))
self.timer.log("end")
if i > 0:
if stop:
break
# print("loss, s", loss_val.numpy(), s.numpy())
self.timer.print()
diff = time.perf_counter() * 1000 - t1
print(
'\nItr: {0} of {1}. Time {2} ms, per itr: {3}: loss {4}\n'.format(
str(i), str(self.max_iterations), str(diff),
str(diff / (i + 1)), str(loss_val.numpy())))
return loss_val
@tf.function
def calculate_gradients(self, s, T_batch, Tinv_batch, calib_batch, I_batch,
alpha_batch, z_batch, q, mu, logvar, network,
prev_loss, count):
with tf.GradientTape(watch_accessed_variables=False) as tape:
tape.watch([z_batch, alpha_batch])
loss_val = self.prediction_error(s, T_batch, Tinv_batch,
calib_batch, I_batch, alpha_batch,
z_batch, q, network)
loss_val += self.g.log_normal_loss(z_batch, mu, logvar)
gradients = tape.gradient(loss_val, [z_batch, alpha_batch])
rel_e = tf.abs(prev_loss - loss_val) / loss_val
s, count, stop = self.evaluate_rel_error(rel_e, s, count)
return gradients, loss_val, s, count, stop
@tf.function
def prediction_error(self, s, T_batch, Tinv_batch, calib_batch, I_batch,
alpha_batch, z_batch, q, network):
P = network.decode(q, z_batch)
D_batch = network.mapToDepth(alpha_batch, P)
self.timer.log("prediction")
error_photometric, error_depth = self.g.evaluate_photogeometric_error(
I_batch, D_batch, T_batch, Tinv_batch, calib_batch, self.angle_th,
alpha_batch)
self.timer.log("error")
loss_val = error_photometric + error_depth
loss_val += tf.reduce_sum(
tf.square(
tf.reduce_mean(network.mapToDepth(tf.ones_like(alpha_batch),
P),
axis=[1, 2, 3]) - 1.0))
return loss_val
@tf.function
def prediction_error_pyramid(self, s, T_batch, Tinv_batch, calib_batch,
I_batch, alpha_batch, z_batch, q, network):
D_batch = network.mapToDepth(alpha_batch, network.decode(q, z_batch))
shape = tf.shape(I_batch)
h = tf.cast(shape[1], dtype=tf.float32)
w = tf.cast(shape[2], dtype=tf.float32)
s_dinv = tf.reshape(tf.linalg.diag([s, s, 1.0, 1.0]), shape=[1, 4, 4])
s_d = tf.reshape(tf.linalg.diag([1.0 / s, 1.0 / s, 1.0, 1.0]),
shape=[1, 4, 4])
T_batch_s = tf.matmul(s_d, T_batch)
Tinv_batch_s = tf.matmul(Tinv_batch, s_dinv)
h_s = tf.cast(tf.math.floordiv(h, s), dtype=tf.float32)
w_s = tf.cast(tf.math.floordiv(w, s), dtype=tf.float32)
I_batch_s = tf.image.resize(I_batch, [h_s, w_s])
D_batch_s = tf.image.resize(D_batch, [h_s, w_s])
self.timer.log("prediction")
error_photometric, error_depth = self.g.evaluate_photogeometric_error(
I_batch_s, D_batch_s, T_batch_s, Tinv_batch_s, calib_batch / s,
self.angle_th)
self.timer.log("error")
loss_val = error_photometric + error_depth
return loss_val
@tf.function(experimental_compile=True)
def evaluate_rel_error(self, rel_e, s, count):
if rel_e < self.termination_crit / s:
count += 1.0
else:
count = 0.0
flag = tf.greater_equal(count, 5.0)
if flag:
s /= 2.0
count = tf.zeros_like(count)
flag = tf.logical_not(flag)
if s < 1.0:
s = tf.ones_like(s)
flag = tf.logical_not(flag)
return s, count, flag
def initialize(self, I, z, alpha, T, Tinv, calib, network):
I_batch = tf.stack(I)
calib_batch = tf.stack(calib)
R = self.g.normalized_points(I_batch, calib_batch)
IR = tf.concat((I_batch, R), axis=-1)
_, mu, logvar = network.encode(IR)
mu_unstacked = tf.unstack(mu)
for i, e in enumerate(mu_unstacked):
if tf.reduce_sum(tf.abs(z[i])) == 0.0:
z[i].assign(mu_unstacked[i])
def train(self, I, z, alpha, T, Tinv, calib, network):
I_batch = tf.stack(I)
T_batch = tf.stack(T)
Tinv_batch = tf.stack(Tinv)
calib_batch = tf.stack(calib)
alpha_batch = tf.stack(alpha)
R = self.g.normalized_points(I_batch, calib_batch)
IR = tf.concat((I_batch, R), axis=-1)
z_batch = tf.stack(z)
P, mu, logvar = network(IR, z_batch)
D_batch = network.mapToDepth(alpha_batch, P)
error_photometric, error_depth = self.g.evaluate_photogeometric_error(
I_batch, D_batch, T_batch, Tinv_batch, calib_batch, self.angle_th,
alpha_batch)
recon_loss = error_photometric + error_depth
loss_val = recon_loss
loss_val += self.g.log_normal_loss(z_batch, mu, logvar)
loss_val += network.sum_losses()
loss_val += tf.reduce_sum(
tf.square(
tf.reduce_mean(network.mapToDepth(tf.ones_like(alpha_batch),
P),
axis=[1, 2, 3]) - 1.0))
return recon_loss, loss_val
def get_graphics(self):
return self.g