number_of_keypoints = dataloader_train.number_of_keypoints # 17 # train dataset dataset_train = dataloader_train.input_fn() dataset_valid = dataloader_valid.input_fn() # validation images val_images, val_heatmaps = dataloader_valid.get_images( 0, batch_size=25) # from 22 index 6 images and 6 labels # ================================================ # =============== build model ==================== # ================================================ from model_provider import get_model model = get_model(model_name=model_name, model_subname=model_subname, number_of_keypoints=number_of_keypoints, config_extra=config_extra) loss_object = tf.keras.losses.MeanSquaredError() optimizer = tf.keras.optimizers.Adam( config_training["learning_rate"], epsilon=config_training["epsilon"]) train_loss = tf.keras.metrics.Mean(name="train_loss") valid_loss = tf.keras.metrics.Mean(name="valid_loss") valid_accuracy = tf.keras.metrics.SparseCategoricalAccuracy( name="valid_accuracy") # ================================================ # ============== train the model ================= # ================================================ num_epochs = config_training["number_of_epoch"] # 550
return LAMBDA * 0.5 * loss if LEARNING_RATE_SCHEDULER == "poly": lrs = K.optimizers.schedules.PolynomialDecay(LEARNING_RATE, decay_steps=EPOCHS, end_learning_rate=1e-8, power=0.8) elif LEARNING_RATE_SCHEDULER == "exp_decay": lrs = K.optimizers.schedules.ExponentialDecay(LEARNING_RATE, decay_steps=EPOCHS, decay_rate=0.5) else: lrs = LEARNING_RATE with mirrored_strategy.scope(): generator_g = get_model("{}_gen".format(MODEL), type="gan") generator_f = get_model("{}_gen".format(MODEL), type="gan") discriminator_x = get_model("{}_disc".format(MODEL), type="gan") discriminator_y = get_model("{}_disc".format(MODEL), type="gan") tmp = tf.cast(tf.random.uniform((1, CROP_HEIGHT, CROP_WIDTH, 3), dtype=tf.float32, minval=0, maxval=1), dtype=tf.float32) generator_g(tmp), generator_f(tmp), discriminator_x(tmp), discriminator_y(tmp) generator_g_optimizer = tf.keras.optimizers.Adam(lrs, beta_1=0.5) generator_f_optimizer = tf.keras.optimizers.Adam(lrs, beta_1=0.5) discriminator_x_optimizer = tf.keras.optimizers.Adam(lrs, beta_1=0.5) discriminator_y_optimizer = tf.keras.optimizers.Adam(lrs, beta_1=0.5) def load_models(models_parent_dir):
data = next(iter(val_dataset)) val_images, val_heatmaps = data['image'], data['heatmap'] # print(val_heatmaps) #print('images', val_images) #print('heatmaps', val_heatmaps) # ================================================ # =============== build model ==================== # ================================================ from model_provider import get_model if strategy is not None: with strategy.scope(): model = get_model(model_name=model_name, model_subname=model_subname, number_of_keypoints=number_of_keypoints, config_extra=config_extra, backbone_name=model_backbone_name) else: model = get_model(model_name=model_name, model_subname=model_subname, number_of_keypoints=number_of_keypoints, config_extra=config_extra, backbone_name=model_backbone_name, input_size=config_dataset['in_height'], weights=None) loss_object = tf.keras.losses.MeanSquaredError() scheduler = tf.keras.optimizers.schedules.ExponentialDecay(initial_learning_rate = config_training["learning_rate"], decay_steps = 5000, decay_rate = config_training['decay_rate']) optimizer = tf.keras.optimizers.Adam(scheduler, epsilon=config_training["epsilon"]) train_loss = tf.keras.metrics.Mean(name="train_loss")
g_lrs = G_LEARNING_RATE d_lrs = D_LEARNING_RATE with mirrored_strategy.scope(): tmp = tf.cast(tf.random.uniform((1, CROP_HEIGHT, CROP_WIDTH, 3), dtype=tf.float32, minval=0, maxval=1), dtype=tf.float32) tmp1 = tf.cast(tf.random.uniform( (1, CROP_HEIGHT, CROP_WIDTH, args.classes), dtype=tf.float32, minval=0, maxval=1), dtype=tf.float32) generator = get_model("{}_gen".format(MODEL), type="gan") discriminator = get_model("{}_disc".format(MODEL), type="gan", classes=args.classes) # discriminator = get_model("unet", type="seg", classes=args.classes + 1, base_channels=64) generator(tmp1), discriminator(tmp) generator_optimizer = tfa.optimizers.MovingAverage( tf.keras.optimizers.Adam(g_lrs, beta_1=0.0, beta_2=0.999), average_decay=0.999) discriminator_optimizer = tf.keras.optimizers.Adam(d_lrs, beta_1=0.0, beta_2=0.999) def load_models(models_parent_dir): assert os.path.exists(
def main(argv=None): assert FLAGS.tfrecord_dir, 'tfrecord_dir not set' assert os.path.exists( FLAGS.tfrecord_dir), '%s is not exist' % FLAGS.tfrecord_dir model = model_provider.get_model() train_dir = os.path.join(FLAGS.train_dir, model.NAME) os.makedirs(train_dir, exist_ok=True) checkpoint_path = os.path.join(train_dir, 'model.ckpt') summary_dir = os.path.join(FLAGS.summary_dir, model.NAME) summary_dir = os.path.join(summary_dir, 'train') os.makedirs(summary_dir, exist_ok=True) global_step = tf.Variable(0, trainable=False) tfrecord_path_list = get_tfrecord_path_list(FLAGS.tfrecord_dir) train_path_list, _ = separate_train_test_data(tfrecord_path_list) dataset = dataset_loader \ .load_dataset(train_path_list, model) \ .shuffle(SHUFFLE_BUFFER_SIZE) \ .repeat() \ .batch(FLAGS.batch_size) iterator = dataset.make_one_shot_iterator() _, label_masks_batch, boxes_batch, image_batch = iterator.get_next() tf.summary.image('images', image_batch, max_outputs=4) feature_map = model.base_layers(image_batch) ssd_logits = model.ssd_layers(feature_map) loss = calc_hnm_loss(boxes_batch, ssd_logits, label_masks_batch, hnm_ratio=FLAGS.hnm_ratio) tf.summary.scalar('loss', loss) lr = FLAGS.learning_rate # lr = tf.train.exponential_decay(lr, global_step, 150000, 0.1, staircase=True) tf.summary.scalar('learning_rate', lr) opt = tf.train.AdamOptimizer(lr) # https://www.tensorflow.org/api_docs/python/tf/contrib/layers/batch_norm # http://mickey24.hatenablog.com/entry/2017/07/07/tensorflow-batch-norm-pitfalls update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_op = opt.minimize(loss, global_step=global_step) saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=3) summary_op = tf.summary.merge_all() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) checkpoint = tf.train.get_checkpoint_state(train_dir) if checkpoint and checkpoint.model_checkpoint_path: saver.restore(sess, checkpoint.model_checkpoint_path) summary_writer = tf.summary.FileWriter(summary_dir, sess.graph) current_step = sess.run(global_step) max_step = FLAGS.step + current_step while (current_step < max_step): current_step, loss_value, _ = sess.run( [global_step, loss, train_op]) if current_step % SAVE_BY_STEPS == 0: print('Step: %d, loss: %.4f' % (current_step, loss_value)) summary_str = sess.run(summary_op) summary_writer.add_summary(summary_str, current_step) saver.save(sess, checkpoint_path, global_step=current_step) open(os.path.join(train_dir, 'complete-%d' % max_step), mode='w').close()
#dataset_valid = strategy.experimental_distribute_dataset(dataset_valid) # validation images val_images, val_heatmaps = dataloader_valid.get_images( 0, batch_size=25 ) if dataloader_valid is not None else None, None # from 22 index 6 images and 6 labels # ================================================ # =============== build model ==================== # ================================================ from model_provider import get_model if strategy is not None: with strategy.scope(): model = get_model(model_name=model_name, model_subname=model_subname, number_of_keypoints=number_of_keypoints, config_extra=config_extra, backbone_name=model_backbone_name) else: model = get_model(model_name=model_name, model_subname=model_subname, number_of_keypoints=number_of_keypoints, config_extra=config_extra, backbone_name=model_backbone_name) loss_object = tf.keras.losses.MeanSquaredError() optimizer = tf.keras.optimizers.Adam(config_training["learning_rate"], epsilon=config_training["epsilon"]) train_loss = tf.keras.metrics.Mean(name="train_loss") valid_loss = tf.keras.metrics.Mean(name="valid_loss") valid_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
axis=None) reduced_disc_loss = mirrored_strategy.reduce(tf.distribute.ReduceOp.MEAN, per_replica_disc_losses, axis=None) while reduced_gen_loss / reduced_disc_loss < balance_ratio: per_replica_gen_losses, per_replica_disc_losses = mirrored_strategy.run( train_step, args=(dist_inputs, True)) reduced_gen_loss = mirrored_strategy.reduce( tf.distribute.ReduceOp.MEAN, per_replica_gen_losses, axis=None) reduced_disc_loss = mirrored_strategy.reduce( tf.distribute.ReduceOp.MEAN, per_replica_disc_losses, axis=None) return reduced_gen_loss, reduced_disc_loss with mirrored_strategy.scope(): generator = get_model(args.model + "_gen", activation=act, type="gan") discriminator = get_model(args.model + "_disc", type="gan") tmp = tf.cast(tf.random.uniform((1, args.height, args.width, 3), dtype=tf.float32, minval=0, maxval=1), dtype=tf.float32) generator(tmp) tmp = tf.cast(tf.random.uniform((1, args.height, args.width, 6), dtype=tf.float32, minval=0, maxval=1), dtype=tf.float32) discriminator(tmp) if args.lr_scheduler == "poly": lrs = K.optimizers.schedules.PolynomialDecay(args.lr,
"--resize_original", action="store_true", default=False, help="If true output images are resized back to input image size") args = args.parse_args() width, height = args.width, args.height img_dir = args.img_dir path = args.model_dir save_dir = args.save_dir model_name, ds_name = get_model_props(args.model_dir) if args.input_resize: model = tf.keras.models.load_model(path) else: model = get_model(model_name, classes=DATASET_DICT[ds_name], in_size=(height, width)) load_model_dynamic(args.model_dir, model) if ds_name == "cityscapes19": args.cs19 = False exts = ["*jpg", "*png", "*jpeg"] imgs = [str(img) for ext in exts for img in pathlib.Path(img_dir).rglob(ext)] def infer(im_path, cmap): im = cv2.imread(im_path)[..., ::-1] height_old, width_old = im.shape[0:2] if args.input_resize:
if LEARNING_RATE_SCHEDULER == "poly": lrs = K.optimizers.schedules.PolynomialDecay(LEARNING_RATE, decay_steps=EPOCHS, end_learning_rate=1e-8, power=0.8) elif LEARNING_RATE_SCHEDULER == "exp_decay": lrs = K.optimizers.schedules.ExponentialDecay(LEARNING_RATE, decay_steps=1e5, decay_rate=0.9) else: lrs = LEARNING_RATE with mirrored_strategy.scope(): generator = get_model("{}_gen".format(MODEL), type="gan") discriminator = get_model("{}_disc".format(MODEL), type="gan") encoder = get_model("{}_enc".format(MODEL), type="gan", gen=generator) mlp = get_model("{}_mlp".format(MODEL), type="gan", units=256, num_patches=256) tmp = tf.cast(tf.random.uniform((1, CROP_HEIGHT, CROP_WIDTH, 3), dtype=tf.float32, minval=0, maxval=1), dtype=tf.float32) generator(tmp), discriminator(tmp), encoder(tmp) generator_optimizer = tf.keras.optimizers.Adam(lrs, beta_1=0.5) discriminator_optimizer = tf.keras.optimizers.Adam(lrs, beta_1=0.5)
d_lrs = D_LEARNING_RATE with mirrored_strategy.scope(): vgg_loss = get_loss(name="VGGLoss") tmp = tf.cast(tf.random.uniform((1, CROP_HEIGHT, CROP_WIDTH, 3), dtype=tf.float32, minval=0, maxval=1), dtype=tf.float32) tmp1 = tf.cast(tf.random.uniform( (1, CROP_HEIGHT, CROP_WIDTH, args.classes), dtype=tf.float32, minval=0, maxval=1), dtype=tf.float32) generator = get_model("{}_gen".format(MODEL), type="gan") discriminator = get_model("{}_disc".format(MODEL), type="gan") encoder = get_model("{}_enc".format(MODEL), type="gan") enc_out = encoder(tmp) generator([enc_out, tmp1]), discriminator([tmp, tmp1]) generator_optimizer = tf.keras.optimizers.Adam(g_lrs, beta_1=0.0, beta_2=0.999) discriminator_optimizer = tf.keras.optimizers.Adam(d_lrs, beta_1=0.0, beta_2=0.999) del tmp, tmp1, enc_out def calc_vgg_loss(real, fake): loss = vgg_loss(real, fake)
def main(argv=None): assert FLAGS.tfrecord_dir, 'tfrecord_dir not set' assert os.path.exists(FLAGS.tfrecord_dir), '%s is not exist' % FLAGS.tfrecord_dir model = model_provider.get_model() batch_size = FLAGS.batch_size train_dir = os.path.join(FLAGS.train_dir, model.NAME) os.makedirs(train_dir, exist_ok=True) summary_dir = os.path.join(FLAGS.summary_dir, model.NAME) summary_dir = os.path.join(summary_dir, 'eval') os.makedirs(summary_dir, exist_ok=True) tfrecord_path_list = get_tfrecord_path_list(FLAGS.tfrecord_dir) _, test_path_list = separate_train_test_data(tfrecord_path_list) sample_count = len(test_path_list) num_iter = math.ceil(sample_count / batch_size) with tf.Graph().as_default() as g: global_step = tf.Variable(0, trainable=False) dataset = dataset_loader \ .load_dataset(test_path_list, model, is_train=False) \ .repeat() \ .batch(batch_size) iterator = dataset.make_one_shot_iterator() _, label_masks_batch, boxes_batch, image_batch = iterator.get_next() feature_map = model.base_layers(image_batch, is_train=False) ssd_logits = model.ssd_layers(feature_map, is_train=False) loss = calc_hnm_loss(boxes_batch, ssd_logits, label_masks_batch, hnm_ratio=FLAGS.hnm_ratio) summary_op = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(summary_dir, g) saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=3) best_loss_value = -1 prev_step = -1 while True: checkpoint = tf.train.get_checkpoint_state(train_dir) if checkpoint is None or checkpoint.model_checkpoint_path is None: print('checkpoint not found.') time.sleep(10) continue prev_step, loss_value = _eval(checkpoint, saver, summary_writer, summary_op, global_step, loss, num_iter, prev_step) if loss_value == -1: time.sleep(10) continue if best_loss_value == -1 or best_loss_value > loss_value: best_loss_value = loss_value print('best_loss_value = %f' % best_loss_value) _backup_checkpoint(checkpoint, best_loss_value) _clear_useless_checkpoints(checkpoint) time.sleep(10)
initial_learning_rate=lr, decay_steps=epochs * total_samples // batch_size, decay_rate=0.9) else: lr_scheduler = lr if optimizer_name == "Adam": optimizer = K.optimizers.Adam(learning_rate=lr_scheduler) elif optimizer_name == "RMSProp": optimizer = K.optimizers.RMSprop(learning_rate=lr_scheduler, momentum=momentum) else: optimizer = K.optimizers.SGD(learning_rate=lr_scheduler, momentum=momentum) model = get_model(model_name, classes=classes, in_size=(args.height, args.width), aux=aux) model(tf.random.uniform((1, args.height, args.width, 3), dtype=tf.float32)) if args.load_model: if os.path.exists(os.path.join(args.load_model, "saved_model.pb")): pretrained_model = K.models.load_model(args.load_model) model.set_weights(pretrained_model.get_weights()) print("Model loaded from {} successfully".format( os.path.basename(args.load_model))) else: print("No file found at {}".format( os.path.join(args.load_model, "saved_model.pb"))) total_steps = 0 step = 0 curr_step = 0
number_of_keypoints = dataloader_train.number_of_keypoints # 17 # train dataset dataset_train = dataloader_train.input_fn() dataset_valid = dataloader_valid.input_fn() # validation images val_images, val_heatmaps = dataloader_valid.get_images( 0, batch_size=25) # from 22 index 6 images and 6 labels # ================================================ # =============== build model ==================== # ================================================ from model_provider import get_model model = get_model(model_name=model_name, model_subname=model_subname, number_of_keypoints=number_of_keypoints) loss_object = tf.keras.losses.MeanSquaredError() optimizer = tf.keras.optimizers.Adam(config_training["learning_rate"], epsilon=config_training["epsilon"]) train_loss = tf.keras.metrics.Mean(name="train_loss") valid_loss = tf.keras.metrics.Mean(name="valid_loss") valid_accuracy = tf.keras.metrics.SparseCategoricalAccuracy( name="valid_accuracy") # ================================================ # ============== train the model ================= # ================================================ num_epochs = config_training["number_of_epoch"] # 550
def call(self, x): h_relu1 = self.slice1(x) h_relu2 = self.slice2(h_relu1) h_relu3 = self.slice3(h_relu2) h_relu4 = self.slice4(h_relu3) h_relu5 = self.slice5(h_relu4) out = [h_relu1, h_relu2, h_relu3, h_relu4, h_relu5] return out def feature_loss(real_list, fake_list): intermediate_loss = 0 for real, fake in zip(real_list, fake_list): for j in range(len(fake) - 1): intermediate_loss += tf.reduce_mean(K.losses.MAE(real[j], fake[j])) return intermediate_loss if __name__ == "__main__": from model_provider import get_model physical_devices = tf.config.experimental.list_physical_devices("GPU") for gpu in physical_devices: tf.config.experimental.set_memory_growth(gpu, True) disc_model = get_model("cyclegan_disc", type="gan") y1 = tf.random.uniform((4, 512, 512, 3)) y2 = tf.random.uniform((4, 512, 512, 3)) wloss = WasserSteinLoss() w = wloss(y2, y1) print(w)