number_of_keypoints = dataloader_train.number_of_keypoints # 17
# train dataset
dataset_train = dataloader_train.input_fn()
dataset_valid = dataloader_valid.input_fn()
# validation images
val_images, val_heatmaps = dataloader_valid.get_images(
0, batch_size=25) # from 22 index 6 images and 6 labels
# ================================================
# =============== build model ====================
# ================================================
from model_provider import get_model
model = get_model(model_name=model_name,
model_subname=model_subname,
number_of_keypoints=number_of_keypoints,
config_extra=config_extra)
loss_object = tf.keras.losses.MeanSquaredError()
optimizer = tf.keras.optimizers.Adam(
config_training["learning_rate"], epsilon=config_training["epsilon"])
train_loss = tf.keras.metrics.Mean(name="train_loss")
valid_loss = tf.keras.metrics.Mean(name="valid_loss")
valid_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
name="valid_accuracy")
# ================================================
# ============== train the model =================
# ================================================
num_epochs = config_training["number_of_epoch"] # 550
return LAMBDA * 0.5 * loss
if LEARNING_RATE_SCHEDULER == "poly":
lrs = K.optimizers.schedules.PolynomialDecay(LEARNING_RATE,
decay_steps=EPOCHS,
end_learning_rate=1e-8, power=0.8)
elif LEARNING_RATE_SCHEDULER == "exp_decay":
lrs = K.optimizers.schedules.ExponentialDecay(LEARNING_RATE,
decay_steps=EPOCHS,
decay_rate=0.5)
else:
lrs = LEARNING_RATE
with mirrored_strategy.scope():
generator_g = get_model("{}_gen".format(MODEL), type="gan")
generator_f = get_model("{}_gen".format(MODEL), type="gan")
discriminator_x = get_model("{}_disc".format(MODEL), type="gan")
discriminator_y = get_model("{}_disc".format(MODEL), type="gan")
tmp = tf.cast(tf.random.uniform((1, CROP_HEIGHT, CROP_WIDTH, 3), dtype=tf.float32, minval=0, maxval=1),
dtype=tf.float32)
generator_g(tmp), generator_f(tmp), discriminator_x(tmp), discriminator_y(tmp)
generator_g_optimizer = tf.keras.optimizers.Adam(lrs, beta_1=0.5)
generator_f_optimizer = tf.keras.optimizers.Adam(lrs, beta_1=0.5)
discriminator_x_optimizer = tf.keras.optimizers.Adam(lrs, beta_1=0.5)
discriminator_y_optimizer = tf.keras.optimizers.Adam(lrs, beta_1=0.5)
def load_models(models_parent_dir):
data = next(iter(val_dataset))
val_images, val_heatmaps = data['image'], data['heatmap']
# print(val_heatmaps)
#print('images', val_images)
#print('heatmaps', val_heatmaps)
# ================================================
# =============== build model ====================
# ================================================
from model_provider import get_model
if strategy is not None:
with strategy.scope():
model = get_model(model_name=model_name,
model_subname=model_subname,
number_of_keypoints=number_of_keypoints,
config_extra=config_extra,
backbone_name=model_backbone_name)
else:
model = get_model(model_name=model_name,
model_subname=model_subname,
number_of_keypoints=number_of_keypoints,
config_extra=config_extra,
backbone_name=model_backbone_name,
input_size=config_dataset['in_height'],
weights=None)
loss_object = tf.keras.losses.MeanSquaredError()
scheduler = tf.keras.optimizers.schedules.ExponentialDecay(initial_learning_rate = config_training["learning_rate"], decay_steps = 5000, decay_rate = config_training['decay_rate'])
optimizer = tf.keras.optimizers.Adam(scheduler, epsilon=config_training["epsilon"])
train_loss = tf.keras.metrics.Mean(name="train_loss")
g_lrs = G_LEARNING_RATE
d_lrs = D_LEARNING_RATE
with mirrored_strategy.scope():
tmp = tf.cast(tf.random.uniform((1, CROP_HEIGHT, CROP_WIDTH, 3),
dtype=tf.float32,
minval=0,
maxval=1),
dtype=tf.float32)
tmp1 = tf.cast(tf.random.uniform(
(1, CROP_HEIGHT, CROP_WIDTH, args.classes),
dtype=tf.float32,
minval=0,
maxval=1),
dtype=tf.float32)
generator = get_model("{}_gen".format(MODEL), type="gan")
discriminator = get_model("{}_disc".format(MODEL),
type="gan",
classes=args.classes)
# discriminator = get_model("unet", type="seg", classes=args.classes + 1, base_channels=64)
generator(tmp1), discriminator(tmp)
generator_optimizer = tfa.optimizers.MovingAverage(
tf.keras.optimizers.Adam(g_lrs, beta_1=0.0, beta_2=0.999),
average_decay=0.999)
discriminator_optimizer = tf.keras.optimizers.Adam(d_lrs,
beta_1=0.0,
beta_2=0.999)
def load_models(models_parent_dir):
assert os.path.exists(
def main(argv=None):
assert FLAGS.tfrecord_dir, 'tfrecord_dir not set'
assert os.path.exists(
FLAGS.tfrecord_dir), '%s is not exist' % FLAGS.tfrecord_dir
model = model_provider.get_model()
train_dir = os.path.join(FLAGS.train_dir, model.NAME)
os.makedirs(train_dir, exist_ok=True)
checkpoint_path = os.path.join(train_dir, 'model.ckpt')
summary_dir = os.path.join(FLAGS.summary_dir, model.NAME)
summary_dir = os.path.join(summary_dir, 'train')
os.makedirs(summary_dir, exist_ok=True)
global_step = tf.Variable(0, trainable=False)
tfrecord_path_list = get_tfrecord_path_list(FLAGS.tfrecord_dir)
train_path_list, _ = separate_train_test_data(tfrecord_path_list)
dataset = dataset_loader \
.load_dataset(train_path_list, model) \
.shuffle(SHUFFLE_BUFFER_SIZE) \
.repeat() \
.batch(FLAGS.batch_size)
iterator = dataset.make_one_shot_iterator()
_, label_masks_batch, boxes_batch, image_batch = iterator.get_next()
tf.summary.image('images', image_batch, max_outputs=4)
feature_map = model.base_layers(image_batch)
ssd_logits = model.ssd_layers(feature_map)
loss = calc_hnm_loss(boxes_batch,
ssd_logits,
label_masks_batch,
hnm_ratio=FLAGS.hnm_ratio)
tf.summary.scalar('loss', loss)
lr = FLAGS.learning_rate
# lr = tf.train.exponential_decay(lr, global_step, 150000, 0.1, staircase=True)
tf.summary.scalar('learning_rate', lr)
opt = tf.train.AdamOptimizer(lr)
# https://www.tensorflow.org/api_docs/python/tf/contrib/layers/batch_norm
# http://mickey24.hatenablog.com/entry/2017/07/07/tensorflow-batch-norm-pitfalls
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = opt.minimize(loss, global_step=global_step)
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=3)
summary_op = tf.summary.merge_all()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
checkpoint = tf.train.get_checkpoint_state(train_dir)
if checkpoint and checkpoint.model_checkpoint_path:
saver.restore(sess, checkpoint.model_checkpoint_path)
summary_writer = tf.summary.FileWriter(summary_dir, sess.graph)
current_step = sess.run(global_step)
max_step = FLAGS.step + current_step
while (current_step < max_step):
current_step, loss_value, _ = sess.run(
[global_step, loss, train_op])
if current_step % SAVE_BY_STEPS == 0:
print('Step: %d, loss: %.4f' % (current_step, loss_value))
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, current_step)
saver.save(sess, checkpoint_path, global_step=current_step)
open(os.path.join(train_dir, 'complete-%d' % max_step),
mode='w').close()
#dataset_valid = strategy.experimental_distribute_dataset(dataset_valid)
# validation images
val_images, val_heatmaps = dataloader_valid.get_images(
0, batch_size=25
) if dataloader_valid is not None else None, None # from 22 index 6 images and 6 labels
# ================================================
# =============== build model ====================
# ================================================
from model_provider import get_model
if strategy is not None:
with strategy.scope():
model = get_model(model_name=model_name,
model_subname=model_subname,
number_of_keypoints=number_of_keypoints,
config_extra=config_extra,
backbone_name=model_backbone_name)
else:
model = get_model(model_name=model_name,
model_subname=model_subname,
number_of_keypoints=number_of_keypoints,
config_extra=config_extra,
backbone_name=model_backbone_name)
loss_object = tf.keras.losses.MeanSquaredError()
optimizer = tf.keras.optimizers.Adam(config_training["learning_rate"],
epsilon=config_training["epsilon"])
train_loss = tf.keras.metrics.Mean(name="train_loss")
valid_loss = tf.keras.metrics.Mean(name="valid_loss")
valid_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
axis=None)
reduced_disc_loss = mirrored_strategy.reduce(tf.distribute.ReduceOp.MEAN,
per_replica_disc_losses,
axis=None)
while reduced_gen_loss / reduced_disc_loss < balance_ratio:
per_replica_gen_losses, per_replica_disc_losses = mirrored_strategy.run(
train_step, args=(dist_inputs, True))
reduced_gen_loss = mirrored_strategy.reduce(
tf.distribute.ReduceOp.MEAN, per_replica_gen_losses, axis=None)
reduced_disc_loss = mirrored_strategy.reduce(
tf.distribute.ReduceOp.MEAN, per_replica_disc_losses, axis=None)
return reduced_gen_loss, reduced_disc_loss
with mirrored_strategy.scope():
generator = get_model(args.model + "_gen", activation=act, type="gan")
discriminator = get_model(args.model + "_disc", type="gan")
tmp = tf.cast(tf.random.uniform((1, args.height, args.width, 3),
dtype=tf.float32,
minval=0,
maxval=1),
dtype=tf.float32)
generator(tmp)
tmp = tf.cast(tf.random.uniform((1, args.height, args.width, 6),
dtype=tf.float32,
minval=0,
maxval=1),
dtype=tf.float32)
discriminator(tmp)
if args.lr_scheduler == "poly":
lrs = K.optimizers.schedules.PolynomialDecay(args.lr,
"--resize_original",
action="store_true",
default=False,
help="If true output images are resized back to input image size")
args = args.parse_args()
width, height = args.width, args.height
img_dir = args.img_dir
path = args.model_dir
save_dir = args.save_dir
model_name, ds_name = get_model_props(args.model_dir)
if args.input_resize:
model = tf.keras.models.load_model(path)
else:
model = get_model(model_name,
classes=DATASET_DICT[ds_name],
in_size=(height, width))
load_model_dynamic(args.model_dir, model)
if ds_name == "cityscapes19":
args.cs19 = False
exts = ["*jpg", "*png", "*jpeg"]
imgs = [str(img) for ext in exts for img in pathlib.Path(img_dir).rglob(ext)]
def infer(im_path, cmap):
im = cv2.imread(im_path)[..., ::-1]
height_old, width_old = im.shape[0:2]
if args.input_resize:
if LEARNING_RATE_SCHEDULER == "poly":
lrs = K.optimizers.schedules.PolynomialDecay(LEARNING_RATE,
decay_steps=EPOCHS,
end_learning_rate=1e-8,
power=0.8)
elif LEARNING_RATE_SCHEDULER == "exp_decay":
lrs = K.optimizers.schedules.ExponentialDecay(LEARNING_RATE,
decay_steps=1e5,
decay_rate=0.9)
else:
lrs = LEARNING_RATE
with mirrored_strategy.scope():
generator = get_model("{}_gen".format(MODEL), type="gan")
discriminator = get_model("{}_disc".format(MODEL), type="gan")
encoder = get_model("{}_enc".format(MODEL), type="gan", gen=generator)
mlp = get_model("{}_mlp".format(MODEL),
type="gan",
units=256,
num_patches=256)
tmp = tf.cast(tf.random.uniform((1, CROP_HEIGHT, CROP_WIDTH, 3),
dtype=tf.float32,
minval=0,
maxval=1),
dtype=tf.float32)
generator(tmp), discriminator(tmp), encoder(tmp)
generator_optimizer = tf.keras.optimizers.Adam(lrs, beta_1=0.5)
discriminator_optimizer = tf.keras.optimizers.Adam(lrs, beta_1=0.5)
d_lrs = D_LEARNING_RATE
with mirrored_strategy.scope():
vgg_loss = get_loss(name="VGGLoss")
tmp = tf.cast(tf.random.uniform((1, CROP_HEIGHT, CROP_WIDTH, 3),
dtype=tf.float32,
minval=0,
maxval=1),
dtype=tf.float32)
tmp1 = tf.cast(tf.random.uniform(
(1, CROP_HEIGHT, CROP_WIDTH, args.classes),
dtype=tf.float32,
minval=0,
maxval=1),
dtype=tf.float32)
generator = get_model("{}_gen".format(MODEL), type="gan")
discriminator = get_model("{}_disc".format(MODEL), type="gan")
encoder = get_model("{}_enc".format(MODEL), type="gan")
enc_out = encoder(tmp)
generator([enc_out, tmp1]), discriminator([tmp, tmp1])
generator_optimizer = tf.keras.optimizers.Adam(g_lrs,
beta_1=0.0,
beta_2=0.999)
discriminator_optimizer = tf.keras.optimizers.Adam(d_lrs,
beta_1=0.0,
beta_2=0.999)
del tmp, tmp1, enc_out
def calc_vgg_loss(real, fake):
loss = vgg_loss(real, fake)
def main(argv=None):
assert FLAGS.tfrecord_dir, 'tfrecord_dir not set'
assert os.path.exists(FLAGS.tfrecord_dir), '%s is not exist' % FLAGS.tfrecord_dir
model = model_provider.get_model()
batch_size = FLAGS.batch_size
train_dir = os.path.join(FLAGS.train_dir, model.NAME)
os.makedirs(train_dir, exist_ok=True)
summary_dir = os.path.join(FLAGS.summary_dir, model.NAME)
summary_dir = os.path.join(summary_dir, 'eval')
os.makedirs(summary_dir, exist_ok=True)
tfrecord_path_list = get_tfrecord_path_list(FLAGS.tfrecord_dir)
_, test_path_list = separate_train_test_data(tfrecord_path_list)
sample_count = len(test_path_list)
num_iter = math.ceil(sample_count / batch_size)
with tf.Graph().as_default() as g:
global_step = tf.Variable(0, trainable=False)
dataset = dataset_loader \
.load_dataset(test_path_list, model, is_train=False) \
.repeat() \
.batch(batch_size)
iterator = dataset.make_one_shot_iterator()
_, label_masks_batch, boxes_batch, image_batch = iterator.get_next()
feature_map = model.base_layers(image_batch, is_train=False)
ssd_logits = model.ssd_layers(feature_map, is_train=False)
loss = calc_hnm_loss(boxes_batch, ssd_logits, label_masks_batch,
hnm_ratio=FLAGS.hnm_ratio)
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(summary_dir, g)
saver = tf.train.Saver(var_list=tf.global_variables(),
max_to_keep=3)
best_loss_value = -1
prev_step = -1
while True:
checkpoint = tf.train.get_checkpoint_state(train_dir)
if checkpoint is None or checkpoint.model_checkpoint_path is None:
print('checkpoint not found.')
time.sleep(10)
continue
prev_step, loss_value = _eval(checkpoint,
saver,
summary_writer, summary_op,
global_step, loss,
num_iter,
prev_step)
if loss_value == -1:
time.sleep(10)
continue
if best_loss_value == -1 or best_loss_value > loss_value:
best_loss_value = loss_value
print('best_loss_value = %f' % best_loss_value)
_backup_checkpoint(checkpoint, best_loss_value)
_clear_useless_checkpoints(checkpoint)
time.sleep(10)
initial_learning_rate=lr,
decay_steps=epochs * total_samples // batch_size,
decay_rate=0.9)
else:
lr_scheduler = lr
if optimizer_name == "Adam":
optimizer = K.optimizers.Adam(learning_rate=lr_scheduler)
elif optimizer_name == "RMSProp":
optimizer = K.optimizers.RMSprop(learning_rate=lr_scheduler,
momentum=momentum)
else:
optimizer = K.optimizers.SGD(learning_rate=lr_scheduler,
momentum=momentum)
model = get_model(model_name,
classes=classes,
in_size=(args.height, args.width),
aux=aux)
model(tf.random.uniform((1, args.height, args.width, 3), dtype=tf.float32))
if args.load_model:
if os.path.exists(os.path.join(args.load_model, "saved_model.pb")):
pretrained_model = K.models.load_model(args.load_model)
model.set_weights(pretrained_model.get_weights())
print("Model loaded from {} successfully".format(
os.path.basename(args.load_model)))
else:
print("No file found at {}".format(
os.path.join(args.load_model, "saved_model.pb")))
total_steps = 0
step = 0
curr_step = 0
number_of_keypoints = dataloader_train.number_of_keypoints # 17
# train dataset
dataset_train = dataloader_train.input_fn()
dataset_valid = dataloader_valid.input_fn()
# validation images
val_images, val_heatmaps = dataloader_valid.get_images(
0, batch_size=25) # from 22 index 6 images and 6 labels
# ================================================
# =============== build model ====================
# ================================================
from model_provider import get_model
model = get_model(model_name=model_name,
model_subname=model_subname,
number_of_keypoints=number_of_keypoints)
loss_object = tf.keras.losses.MeanSquaredError()
optimizer = tf.keras.optimizers.Adam(config_training["learning_rate"],
epsilon=config_training["epsilon"])
train_loss = tf.keras.metrics.Mean(name="train_loss")
valid_loss = tf.keras.metrics.Mean(name="valid_loss")
valid_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
name="valid_accuracy")
# ================================================
# ============== train the model =================
# ================================================
num_epochs = config_training["number_of_epoch"] # 550
def call(self, x):
h_relu1 = self.slice1(x)
h_relu2 = self.slice2(h_relu1)
h_relu3 = self.slice3(h_relu2)
h_relu4 = self.slice4(h_relu3)
h_relu5 = self.slice5(h_relu4)
out = [h_relu1, h_relu2, h_relu3, h_relu4, h_relu5]
return out
def feature_loss(real_list, fake_list):
intermediate_loss = 0
for real, fake in zip(real_list, fake_list):
for j in range(len(fake) - 1):
intermediate_loss += tf.reduce_mean(K.losses.MAE(real[j], fake[j]))
return intermediate_loss
if __name__ == "__main__":
from model_provider import get_model
physical_devices = tf.config.experimental.list_physical_devices("GPU")
for gpu in physical_devices:
tf.config.experimental.set_memory_growth(gpu, True)
disc_model = get_model("cyclegan_disc", type="gan")
y1 = tf.random.uniform((4, 512, 512, 3))
y2 = tf.random.uniform((4, 512, 512, 3))
wloss = WasserSteinLoss()
w = wloss(y2, y1)
print(w)