history_teacher = LossAccHistory() training = KDModel.NormalTraining(teacher_model) teacher_model.summary() # plot_model(teacher_model, show_shapes=True, to_file='teacher_model.png') for epoch in range(1, EPOCHS_T + 1): loss_metric = Mean() loss_metric_val = Mean() acc_metric = Mean() acc_metric_val = Mean() acc = CategoricalAccuracy() acc_val = CategoricalAccuracy() # 各バッチごとに学習 for (x_train_, y_train_), (x_val_, y_val_) in ds: loss_value, grads = training.grad(x_train_, y_train_) optimizer.apply_gradients(zip(grads, teacher_model.trainable_weights)) loss_value_test = training.loss(x_val_, y_val_) probs = tf.nn.softmax(teacher_model(x_train_)) probs_val = tf.nn.softmax(teacher_model(x_val_)) loss_metric(loss_value) acc_metric(acc(y_train_, probs)) loss_metric_val(loss_value_test) acc_metric_val(acc_val(y_val_, probs_val)) # 学習進捗の表示 print( 'Epoch {}/{}: Loss: {:.3f}, Accuracy: {:.3%}, Validation Loss: {:.3f}, Validation Accuracy: {:.3%}' .format(epoch, EPOCHS_T, loss_metric.result().numpy(), acc_metric.result().numpy(),
class Trainer: def __init__(self, config): self.predict_only = config['Trainer']['predict_only'] self.nbr_samples_per_update = config['Trainer']['nbr_samples_per_update'] self.optimizer = Adamax(lr=1e-3) self.buffered_samples = [] def run(self, sample, network): loss = -tf.constant(float('Inf'), dtype=tf.float32) recon_loss = -tf.constant(float('Inf'), dtype=tf.float32) if sample is not None: if self.predict_only: loss = sample.predict(network) return loss, recon_loss else: self.buffered_samples.append(sample) if len(self.buffered_samples) >= self.nbr_samples_per_update: trainable_variables = network.getTrainableVariables() for sample in self.buffered_samples: alpha = sample.getAlpha() trainable_variables = [ *trainable_variables, *alpha] z = sample.getZ() trainable_variables = [ *trainable_variables, *z] with tf.GradientTape(watch_accessed_variables=False) as tape: tape.watch(trainable_variables) loss = tf.constant(0.0, dtype=tf.float32) recon_loss = tf.constant(0.0, dtype=tf.float32) for sample in self.buffered_samples: recon_loss_i, loss_i = sample.train( network) loss += loss_i recon_loss += recon_loss_i gradients = tape.gradient( loss, trainable_variables) self.optimizer.apply_gradients( zip(gradients, trainable_variables)) loss /= len(self.buffered_samples) recon_loss /= len(self.buffered_samples) self.buffered_samples.clear() return loss, recon_loss
class DepthOptimizer: def __init__(self, config): self.optimizer = Adamax(lr=1e-3) self.g = Graphics3() def get_graphics(self): return self.g def updateLearningRate(self, lr): self.optimizer.learning_rate.assign(lr) def getLearningRate(self): return self.optimizer.learning_rate def getCheckPointVariables(self): return {"depth_optimizer": self.optimizer} def predict(self, I, D, mask, z, alpha, calib, network): I_batch = tf.stack(I) D_batch = tf.stack(D) mask_batch = tf.stack(mask) calib_batch = tf.stack(calib) R = self.g.normalized_points(I_batch, calib_batch) IR = tf.concat((I_batch, R), axis=-1) q, mu, logvar = network.encode(IR) trainable_variables = z rel_error = tf.constant(0.0, dtype=tf.float32) for i in range(200): with tf.GradientTape(watch_accessed_variables=False) as tape: tape.watch(trainable_variables) z_batch = tf.stack(z) alpha_batch = tf.stack(alpha) loss_next = self.predict_update(D_batch, mask_batch, q, z_batch, alpha_batch, network) loss_next += self.g.log_normal_loss(z_batch, mu, logvar) gradients = tape.gradient(loss_next, trainable_variables) self.optimizer.apply_gradients( zip(gradients, trainable_variables)) if i > 0: rel_error = 0.8*(tf.abs(loss_prev-loss_next) / loss_next) + 0.2 * rel_error if i > 0 and rel_error < 1e-3: break loss_prev = loss_next return loss_next @tf.function(experimental_compile=False) def predict_update(self, D_batch, mask_batch, q, z_batch, alpha_batch, network): shape = tf.shape(D_batch) D_p = network.decode(q, z_batch) loss_val = self.loss(D_batch, mask_batch, D_p, alpha_batch) return loss_val def train(self, I, D, mask, z, alpha, calib, network): I_batch = tf.stack(I) D_batch = tf.stack(D) mask_batch = tf.stack(mask) alpha_batch = tf.stack(alpha) z_batch = tf.stack(z) recon_loss, loss_val = self.train_loss( I_batch, D_batch, mask_batch, z_batch, alpha_batch, calib, network) return recon_loss, loss_val @tf.function(experimental_compile=False) def train_loss(self, I_batch, D_batch, mask_batch, z_batch, alpha_batch, calib, network): shape = tf.shape(I_batch) calib_batch = tf.stack(calib) R = self.g.normalized_points(I_batch, calib_batch) IR = tf.concat((I_batch, R), axis=-1) P, mu, logvar = network(IR, z_batch) D_p = network.mapToDepth(alpha_batch, P) recon_loss = self.loss(D_batch, mask_batch, D_p, alpha_batch) reg_loss = network.sum_losses() loss_val = recon_loss + reg_loss loss_val += self.g.log_normal_loss(z_batch, mu, logvar) return recon_loss, loss_val def mapToProximal(self, A, Dinv): shape = tf.shape(Dinv) A = tf.reshape(A, shape=[shape[0], 1, 1, 1]) return tf.math.divide_no_nan(A*tf.ones_like(Dinv), Dinv + A*tf.ones_like(Dinv)) @tf.function(experimental_compile=False) def loss(self, y_true, mask, y_pred, alpha_batch): return tf.reduce_sum(tf.boolean_mask(tf.math.square(y_true-y_pred), mask))/0.01 @tf.function(experimental_compile=False) def loss2(self, y_true, mask, y_pred, alpha_batch): Diff = y_true-y_pred shape = tf.shape(Diff) Diff = Diff/tf.reshape(alpha_batch, shape=[shape[0], 1, 1, 1]) loss = tf.boolean_mask(self.g.Huber(Diff, 0.1), mask) return tf.reduce_sum(loss)/0.01
class PhotometricOptimizer2: def __init__(self, config): self.max_iterations = config['PhotometricOptimizer']['max_iterations'] self.termination_crit = config['PhotometricOptimizer'][ 'termination_crit'] self.image_height = config['dataset']['image_height'] self.image_width = config['dataset']['image_width'] self.g = Graphics3() self.optimizer = Adamax(lr=1e-3) self.timer = Timer(config) self.angle_th = np.cos( (config['PhotometricOptimizer']['angle_th'] / 180.0) * np.pi) self.angle_th = tf.constant(self.angle_th, dtype=tf.float32) def updateLearningRate(self, lr): self.optimizer.learning_rate.assign(lr) def getLearningRate(self): return self.optimizer.learning_rate def getCheckPointVariables(self): return {"photometric_optimizer": self.optimizer} def predict(self, I, z, alpha, T, Tinv, calib, network): t1 = time.perf_counter() * 1000 self.timer.start() I_batch = tf.stack(I) T_batch = tf.stack(T) Tinv_batch = tf.stack(Tinv) calib_batch = tf.stack(calib) alpha_batch = tf.stack(alpha) self.timer.log("stack") R = self.g.normalized_points(I_batch, calib_batch) IR = tf.concat((I_batch, R), axis=-1) q, mu, logvar = network.encode(IR) self.timer.log("setup") s = tf.constant(1.0, dtype=tf.float32) count = tf.constant(0.0, dtype=tf.float32) loss_val = tf.constant(-10.0, dtype=tf.float32) mu_unstacked = tf.unstack(mu) sig_unstacked = tf.unstack(tf.math.exp(logvar)) trainable_variables = [*z, *alpha] for i, e in enumerate(mu_unstacked): if tf.reduce_sum(tf.abs(z[i])) == 0.0: z[i].assign(mu_unstacked[i]) for i in range(self.max_iterations): z_batch = tf.stack(z) self.timer.log("gradient") gradients, loss_val, s, count, stop = self.calculate_gradients( s, T_batch, Tinv_batch, calib_batch, I_batch, alpha_batch, z_batch, q, mu, logvar, network, loss_val, count) self.timer.log("update") g = [tf.unstack(gradient) for gradient in gradients] flat_list = [item for sublist in g for item in sublist] self.optimizer.apply_gradients(zip(flat_list, trainable_variables)) self.timer.log("end") if i > 0: if stop: break # print("loss, s", loss_val.numpy(), s.numpy()) self.timer.print() diff = time.perf_counter() * 1000 - t1 print( '\nItr: {0} of {1}. Time {2} ms, per itr: {3}: loss {4}\n'.format( str(i), str(self.max_iterations), str(diff), str(diff / (i + 1)), str(loss_val.numpy()))) return loss_val @tf.function def calculate_gradients(self, s, T_batch, Tinv_batch, calib_batch, I_batch, alpha_batch, z_batch, q, mu, logvar, network, prev_loss, count): with tf.GradientTape(watch_accessed_variables=False) as tape: tape.watch([z_batch, alpha_batch]) loss_val = self.prediction_error(s, T_batch, Tinv_batch, calib_batch, I_batch, alpha_batch, z_batch, q, network) loss_val += self.g.log_normal_loss(z_batch, mu, logvar) gradients = tape.gradient(loss_val, [z_batch, alpha_batch]) rel_e = tf.abs(prev_loss - loss_val) / loss_val s, count, stop = self.evaluate_rel_error(rel_e, s, count) return gradients, loss_val, s, count, stop @tf.function def prediction_error(self, s, T_batch, Tinv_batch, calib_batch, I_batch, alpha_batch, z_batch, q, network): P = network.decode(q, z_batch) D_batch = network.mapToDepth(alpha_batch, P) self.timer.log("prediction") error_photometric, error_depth = self.g.evaluate_photogeometric_error( I_batch, D_batch, T_batch, Tinv_batch, calib_batch, self.angle_th, alpha_batch) self.timer.log("error") loss_val = error_photometric + error_depth loss_val += tf.reduce_sum( tf.square( tf.reduce_mean(network.mapToDepth(tf.ones_like(alpha_batch), P), axis=[1, 2, 3]) - 1.0)) return loss_val @tf.function def prediction_error_pyramid(self, s, T_batch, Tinv_batch, calib_batch, I_batch, alpha_batch, z_batch, q, network): D_batch = network.mapToDepth(alpha_batch, network.decode(q, z_batch)) shape = tf.shape(I_batch) h = tf.cast(shape[1], dtype=tf.float32) w = tf.cast(shape[2], dtype=tf.float32) s_dinv = tf.reshape(tf.linalg.diag([s, s, 1.0, 1.0]), shape=[1, 4, 4]) s_d = tf.reshape(tf.linalg.diag([1.0 / s, 1.0 / s, 1.0, 1.0]), shape=[1, 4, 4]) T_batch_s = tf.matmul(s_d, T_batch) Tinv_batch_s = tf.matmul(Tinv_batch, s_dinv) h_s = tf.cast(tf.math.floordiv(h, s), dtype=tf.float32) w_s = tf.cast(tf.math.floordiv(w, s), dtype=tf.float32) I_batch_s = tf.image.resize(I_batch, [h_s, w_s]) D_batch_s = tf.image.resize(D_batch, [h_s, w_s]) self.timer.log("prediction") error_photometric, error_depth = self.g.evaluate_photogeometric_error( I_batch_s, D_batch_s, T_batch_s, Tinv_batch_s, calib_batch / s, self.angle_th) self.timer.log("error") loss_val = error_photometric + error_depth return loss_val @tf.function(experimental_compile=True) def evaluate_rel_error(self, rel_e, s, count): if rel_e < self.termination_crit / s: count += 1.0 else: count = 0.0 flag = tf.greater_equal(count, 5.0) if flag: s /= 2.0 count = tf.zeros_like(count) flag = tf.logical_not(flag) if s < 1.0: s = tf.ones_like(s) flag = tf.logical_not(flag) return s, count, flag def initialize(self, I, z, alpha, T, Tinv, calib, network): I_batch = tf.stack(I) calib_batch = tf.stack(calib) R = self.g.normalized_points(I_batch, calib_batch) IR = tf.concat((I_batch, R), axis=-1) _, mu, logvar = network.encode(IR) mu_unstacked = tf.unstack(mu) for i, e in enumerate(mu_unstacked): if tf.reduce_sum(tf.abs(z[i])) == 0.0: z[i].assign(mu_unstacked[i]) def train(self, I, z, alpha, T, Tinv, calib, network): I_batch = tf.stack(I) T_batch = tf.stack(T) Tinv_batch = tf.stack(Tinv) calib_batch = tf.stack(calib) alpha_batch = tf.stack(alpha) R = self.g.normalized_points(I_batch, calib_batch) IR = tf.concat((I_batch, R), axis=-1) z_batch = tf.stack(z) P, mu, logvar = network(IR, z_batch) D_batch = network.mapToDepth(alpha_batch, P) error_photometric, error_depth = self.g.evaluate_photogeometric_error( I_batch, D_batch, T_batch, Tinv_batch, calib_batch, self.angle_th, alpha_batch) recon_loss = error_photometric + error_depth loss_val = recon_loss loss_val += self.g.log_normal_loss(z_batch, mu, logvar) loss_val += network.sum_losses() loss_val += tf.reduce_sum( tf.square( tf.reduce_mean(network.mapToDepth(tf.ones_like(alpha_batch), P), axis=[1, 2, 3]) - 1.0)) return recon_loss, loss_val def get_graphics(self): return self.g