def build_model(self):
""" Graph Input """
# images
Image_Data_Class = ImageData(self.img_size, self.c_dim, self.custom_dataset)
inputs = tf.data.Dataset.from_tensor_slices(self.data)
gpu_device = '/gpu:0'
inputs = inputs.\
apply(shuffle_and_repeat(self.dataset_num)).\
apply(map_and_batch(Image_Data_Class.image_processing, self.batch_size, num_parallel_batches=16, drop_remainder=True)).\
apply(prefetch_to_device(gpu_device, self.batch_size))
inputs_iterator = inputs.make_one_shot_iterator()
self.inputs = inputs_iterator.get_next()
# noises
self.z = tf.truncated_normal(shape=[self.batch_size, 1, 1, self.z_dim], name='random_z')
""" Loss Function """
# output of D for real images
real_logits = self.discriminator(self.inputs)
# output of D for fake images
fake_images = self.generator(self.z)
fake_logits = self.discriminator(fake_images, reuse=True)
if self.gan_type.__contains__('wgan') or self.gan_type == 'dragan':
GP = self.gradient_penalty(real=self.inputs, fake=fake_images)
else:
GP = 0
# get loss for discriminator
self.d_loss = discriminator_loss(self.gan_type, real=real_logits, fake=fake_logits) + GP
# get loss for generator
self.g_loss = generator_loss(self.gan_type, fake=fake_logits)
""" Training """
# divide trainable variables into a group for D and a group for G
t_vars = tf.trainable_variables()
d_vars = [var for var in t_vars if 'discriminator' in var.name]
g_vars = [var for var in t_vars if 'generator' in var.name]
# optimizers
with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
self.d_optim = tf.train.AdamOptimizer(self.d_learning_rate, beta1=self.beta1, beta2=self.beta2).minimize(self.d_loss, var_list=d_vars)
self.opt = MovingAverageOptimizer(tf.train.AdamOptimizer(self.g_learning_rate, beta1=self.beta1, beta2=self.beta2), average_decay=self.moving_decay)
self.g_optim = self.opt.minimize(self.g_loss, var_list=g_vars)
"""" Testing """
# for test
self.fake_images = self.generator(self.z, is_training=False, reuse=True)
""" Summary """
self.d_sum = tf.summary.scalar("d_loss", self.d_loss)
self.g_sum = tf.summary.scalar("g_loss", self.g_loss)
def testMovingAverageOptimizer(self):
with self.test_session() as sess:
global_step = tf.train.get_or_create_global_step()
x = tf.Variable(50, dtype=tf.float32, name="x")
y = tf.Variable(50, dtype=tf.float32, name="y")
tower_grads = []
opt = get_optimizer(global_step,
optimizer="GD",
learning_rate=1.0,
batch_size=1,
num_epochs_per_decay=100,
example_size=1,
learn_rate_decay_factor=0.2,
min_learn_rate=1e-5)
opt = MovingAverageOptimizer(opt, average_decay=0.8)
for i in range(2):
with tf.device("/cpu:{}".format(0)):
with tf.name_scope("cpu_{}".format(i)):
loss = tf.pow(x - 10.0, 2) + 9.0 + tf.pow(y - 5., 2)
loss = tf.reduce_sum(loss)
tf.losses.add_loss(loss)
grads, _, _ = get_train_opv3(optimizer=opt, loss=loss)
tower_grads.append(grads)
avg_grads = average_grads(tower_grads, clip_norm=1)
opt0 = apply_gradientsv3(avg_grads, global_step, opt)
g_x = avg_grads[0][0]
g_y = avg_grads[1][0]
opt1 = get_batch_norm_ops()
train_op = tf.group(opt0, opt1)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for i in range(60):
r_x, r_y, rgx, rgy, l, _ = sess.run(
[x, y, g_x, g_y, loss, train_op])
print(f"step {i}: ", r_x, r_y, rgx, rgy, l)
saver = opt.swapping_saver()
check_point_dir = tempfile.gettempdir()
ckpt_path = os.path.join(check_point_dir, CHECK_POINT_FILE_NAME)
saver.save(sess, ckpt_path, global_step=1)
def tpu_model_fn(self, features, labels, mode, params):
params = EasyDict(**params)
d_loss, d_vars, g_loss, g_vars, fake_images, fake_logits, z = self.base_model_fn(
features, labels, mode, params)
# --------------------------------------------------------------------------
# Predict
# --------------------------------------------------------------------------
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
"z": z,
"fake_image": fake_images,
"fake_logits": fake_logits,
}
return tf.contrib.tpu.TPUEstimatorSpec(mode,
predictions=predictions)
# --------------------------------------------------------------------------
# Train or Eval
# --------------------------------------------------------------------------
loss = g_loss
for i in range(params.n_critic):
loss += d_loss
if mode == tf.estimator.ModeKeys.EVAL:
# Hack to allow it out of a fixed batch size TPU
d_loss_batched = tf.tile(tf.expand_dims(d_loss, 0),
[params.batch_size])
g_loss_batched = tf.tile(tf.expand_dims(g_loss, 0),
[params.batch_size])
return tf.contrib.tpu.TPUEstimatorSpec(
mode=mode,
loss=loss,
eval_metrics=(lambda d_loss, g_loss, fake_logits: self.
tpu_metric_fn(d_loss, g_loss, fake_logits),
[d_loss_batched, g_loss_batched, fake_logits]))
if mode == tf.estimator.ModeKeys.TRAIN:
# Create training ops for both D and G
d_optimizer = tf.train.AdamOptimizer(params.d_lr,
beta1=params.beta1,
beta2=params.beta2)
if params.use_tpu:
d_optimizer = tf.contrib.tpu.CrossShardOptimizer(d_optimizer)
d_train_op = d_optimizer.minimize(
d_loss,
var_list=d_vars,
global_step=tf.train.get_global_step())
g_optimizer = MovingAverageOptimizer(
tf.train.AdamOptimizer(params.g_lr,
beta1=params.beta1,
beta2=params.beta2),
average_decay=params.moving_decay)
if params.use_tpu:
g_optimizer = tf.contrib.tpu.CrossShardOptimizer(g_optimizer)
g_train_op = g_optimizer.minimize(
g_loss,
var_list=g_vars,
global_step=tf.train.get_global_step())
# For each training op of G, do n_critic training ops of D
train_ops = [g_train_op]
for i in range(params.n_critic):
train_ops.append(d_train_op)
train_op = tf.group(*train_ops)
return tf.contrib.tpu.TPUEstimatorSpec(mode,
loss=loss,
train_op=train_op)