np.random.seed()
iteration = sess.run(global_step)
is_img = tf.placeholder(tf.float32, shape=[None, None, None, 3])
is_feat = tf.nn.softmax(
tf.contrib.gan.eval.run_inception(
tf.image.resize_bilinear(is_img, [299, 299])))
if os.path.exists('./snapshots/best_model.ckpt.index'):
saver.restore(sess, './snapshots/best_model.ckpt')
sample_images = sess.run(x_hat,
feed_dict={
z: sample_noise,
is_training: False
})
save_images(sample_images, 'tmp/{:06d}.png'.format(iteration))
# Sample 50000 images for evaluation
print("Evaluating...")
num_images_to_eval = 50000
eval_images = []
num_batches = num_images_to_eval // FLAGS.batch_size + 1
print("Calculating Inception Score. Sampling {} images...".format(
num_images_to_eval))
np.random.seed(0)
for _ in range(num_batches):
images = sess.run(x_hat,
feed_dict={
z: generator.generate_noise(),
is_training: False
})
eval_images.append(images)
if (iteration +
1) % FLAGS.display_interval == 0 and not is_start_iteration:
summary_writer.add_summary(summaries, global_step=iteration)
stop = timeit.default_timer()
print('Iter {}: d_loss = {:4f}, g_loss = {:4f}, time = {:2f}s'.format(
iteration, d_loss_curr, g_loss_curr, stop - start))
start = stop
if (iteration +
1) % FLAGS.snapshot_interval == 0 and not is_start_iteration:
saver.save(sess,
output_dir + '/snapshots/model.ckpt',
global_step=iteration)
if y_dim:
sample_images = sess.run(x_hat,
feed_dict={
z: sample_z,
y: sample_y,
is_training: False
})
else:
sample_images = sess.run(x_hat,
feed_dict={
z: sample_z,
is_training: False
})
save_images(sample_images,
output_dir + '/{:06d}.png'.format(iteration))
iteration += 1
is_start_iteration = False
if (iteration +
1) % FLAGS.display_interval == 0 and not is_start_iteration:
summary_writer.add_summary(summaries, global_step=iteration)
stop = timeit.default_timer()
print('Iter {}: d_loss = {:4f}, g_loss = {:4f}, time = {:2f}s'.format(
iteration, d_loss_curr, g_loss_curr, stop - start))
start = stop
if (iteration +
1) % FLAGS.snapshot_interval == 0 and not is_start_iteration:
saver.save(sess, snapshot_dir + '/model.ckpt', global_step=iteration)
sample_images = sess.run(x_hat,
feed_dict={
z: sample_noise,
is_training: False
})
save_images(sample_images, tmp_dir + '/{:06d}.png'.format(iteration))
if (iteration + 1) % FLAGS.evaluation_interval == 0:
sample_images = sess.run(x_hat,
feed_dict={
z: sample_noise,
is_training: False
})
save_images(sample_images, tmp_dir + '{:06d}.png'.format(iteration))
# Sample 50000 images for evaluation
print("Evaluating...")
num_images_to_eval = 50000
eval_images = []
num_batches = num_images_to_eval // FLAGS.batch_size + 1
print("Calculating Inception Score. Sampling {} images...".format(
num_images_to_eval))
np.random.seed(0)
else:
print("Unable to load tensorflow checkpoint for " + FLAGS.data_name)
exit()
print("---")
print(
"For '" + FLAGS.data_name +
"', loaded tensorflow checkpoint from iteration #", sess.run(global_step))
print("> Generating", FLAGS.batch_size, "images")
sample_z = generator.generate_noise()
if y_dim:
sample_images = sess.run(x_hat,
feed_dict={
z: sample_z,
y: sample_y,
is_training: False
})
else:
sample_images = sess.run(x_hat,
feed_dict={
z: sample_z,
is_training: False
})
if save_individual_images:
for i, image in enumerate(sample_images):
scipy.misc.imsave(output_dir + "/" + str(i) + ".png", image)
if save_collage:
save_images(sample_images, output_dir + '/collage.png')
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
best_saver = tf.train.Saver()
is_img = tf.placeholder(tf.float32, shape=[None, None, None, 3])
is_feat = tf.nn.softmax(tf.contrib.gan.eval.run_inception(tf.image.resize_bilinear(is_img, [299, 299])))
if os.path.exists('./snapshots/best_model.ckpt.index'):
saver.restore(sess, './snapshots/best_model.ckpt')
sample_images = sess.run(x_hat, feed_dict={z: sample_noise, is_training: False})
# save_images(sample_images, 'tmp/{:06d}.png'.format(iteration))
save_images(sample_images, 'tmp/test.png')
# Sample 50000 images for evaluation
print("Evaluating...")
num_images_to_eval = 50000
eval_images = []
num_batches = num_images_to_eval // FLAGS.batch_size + 1
print("Calculating Inception Score. Sampling {} images...".format(num_images_to_eval))
np.random.seed(0)
for _ in range(num_batches):
images = sess.run(x_hat, feed_dict={z: generator.generate_noise(), is_training: False})
eval_images.append(images)
np.random.seed()
eval_images = np.vstack(eval_images)
eval_images = eval_images[:num_images_to_eval]
def main(_):
if not os.path.exists(FLAGS.logdir):
os.makedirs(FLAGS.logdir)
# Random seed
rng = np.random.RandomState(FLAGS.seed) # seed labels
rng_data = np.random.RandomState(FLAGS.seed_data) # seed shuffling
# load CIFAR-10
trainx, trainy = cifar10_input._get_dataset(FLAGS.data_dir,
'train') # float [-1 1] images
testx, testy = cifar10_input._get_dataset(FLAGS.data_dir, 'test')
trainx_unl = trainx.copy()
trainx_unl2 = trainx.copy()
nr_batches_train = int(trainx.shape[0] / FLAGS.batch_size)
nr_batches_test = int(testx.shape[0] / FLAGS.batch_size)
# select labeled data
inds = rng_data.permutation(trainx.shape[0])
trainx = trainx[inds]
trainy = trainy[inds]
txs = []
tys = []
for j in range(10):
txs.append(trainx[trainy == j][:FLAGS.labeled])
tys.append(trainy[trainy == j][:FLAGS.labeled])
txs = np.concatenate(txs, axis=0)
tys = np.concatenate(tys, axis=0)
print("Data:")
print('train examples %d, batch %d, test examples %d, batch %d' \
% (trainx.shape[0], nr_batches_train, testx.shape[0], nr_batches_test))
print('histogram train', np.histogram(trainy, bins=10)[0])
print('histogram test ', np.histogram(testy, bins=10)[0])
print("histogram labeled", np.histogram(tys, bins=10)[0])
print("")
'''construct graph'''
print('constructing graph')
unl = tf.placeholder(tf.float32, [FLAGS.batch_size, 32, 32, 3],
name='unlabeled_data_input_pl')
is_training_pl = tf.placeholder(tf.bool, [], name='is_training_pl')
inp = tf.placeholder(tf.float32, [FLAGS.batch_size, 32, 32, 3],
name='labeled_data_input_pl')
lbl = tf.placeholder(tf.int32, [FLAGS.batch_size], name='lbl_input_pl')
# scalar pl
lr_pl = tf.placeholder(tf.float32, [], name='learning_rate_pl')
acc_train_pl = tf.placeholder(tf.float32, [], 'acc_train_pl')
acc_test_pl = tf.placeholder(tf.float32, [], 'acc_test_pl')
acc_test_pl_ema = tf.placeholder(tf.float32, [], 'acc_test_pl')
kl_weight = tf.placeholder(tf.float32, [], 'kl_weight')
random_z = tf.random_uniform([FLAGS.batch_size, 100], name='random_z')
perturb = tf.random_normal([FLAGS.batch_size, 100], mean=0, stddev=0.01)
random_z_pert = random_z + FLAGS.scale * perturb / (
tf.expand_dims(tf.norm(perturb, axis=1), axis=1) * tf.ones([1, 100]))
generator(random_z, is_training_pl,
init=True) # init of weightnorm weights
gen_inp = generator(random_z, is_training_pl, init=False, reuse=True)
gen_inp_pert = generator(random_z_pert,
is_training_pl,
init=False,
reuse=True)
discriminator(unl, is_training_pl, init=True)
logits_lab, _ = discriminator(inp, is_training_pl, init=False, reuse=True)
logits_gen, layer_fake = discriminator(gen_inp,
is_training_pl,
init=False,
reuse=True)
logits_unl, layer_real = discriminator(unl,
is_training_pl,
init=False,
reuse=True)
logits_gen_perturb, layer_fake_perturb = discriminator(gen_inp_pert,
is_training_pl,
init=False,
reuse=True)
with tf.name_scope('loss_functions'):
l_unl = tf.reduce_logsumexp(logits_unl, axis=1)
l_gen = tf.reduce_logsumexp(logits_gen, axis=1)
# discriminator
loss_lab = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(labels=lbl,
logits=logits_lab))
loss_unl = - 0.5 * tf.reduce_mean(l_unl) \
+ 0.5 * tf.reduce_mean(tf.nn.softplus(l_unl)) \
+ 0.5 * tf.reduce_mean(tf.nn.softplus(l_gen))
# generator
m1 = tf.reduce_mean(layer_real, axis=0)
m2 = tf.reduce_mean(layer_fake, axis=0)
j_loss = tf.reduce_mean(
tf.reduce_sum(tf.square(logits_gen - logits_gen_perturb), axis=1))
if FLAGS.nabla:
loss_dis = FLAGS.unl_weight * loss_unl + FLAGS.lbl_weight * loss_lab + kl_weight * j_loss
loss_gen = tf.reduce_mean(tf.abs(m1 - m2))
print('manifold reg enabled')
else:
loss_dis = FLAGS.unl_weight * loss_unl + FLAGS.lbl_weight * loss_lab
loss_gen = tf.reduce_mean(tf.abs(m1 - m2))
correct_pred = tf.equal(tf.cast(tf.argmax(logits_lab, 1), tf.int32),
tf.cast(lbl, tf.int32))
accuracy_classifier = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
with tf.name_scope('optimizers'):
# control op dependencies for batch norm and trainable variables
tvars = tf.trainable_variables()
dvars = [var for var in tvars if 'discriminator_model' in var.name]
gvars = [var for var in tvars if 'generator_model' in var.name]
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
update_ops_gen = [
x for x in update_ops if ('generator_model' in x.name)
]
update_ops_dis = [
x for x in update_ops if ('discriminator_model' in x.name)
]
optimizer_dis = tf.train.AdamOptimizer(learning_rate=lr_pl,
beta1=0.5,
name='dis_optimizer')
optimizer_gen = tf.train.AdamOptimizer(learning_rate=lr_pl,
beta1=0.5,
name='gen_optimizer')
with tf.control_dependencies(update_ops_gen):
train_gen_op = optimizer_gen.minimize(loss_gen, var_list=gvars)
dis_op = optimizer_dis.minimize(loss_dis, var_list=dvars)
ema = tf.train.ExponentialMovingAverage(decay=FLAGS.ma_decay)
maintain_averages_op = ema.apply(dvars)
with tf.control_dependencies([dis_op]):
train_dis_op = tf.group(maintain_averages_op)
logits_ema, _ = discriminator(inp,
is_training_pl,
getter=get_getter(ema),
reuse=True)
correct_pred_ema = tf.equal(
tf.cast(tf.argmax(logits_ema, 1), tf.int32),
tf.cast(lbl, tf.int32))
accuracy_ema = tf.reduce_mean(tf.cast(correct_pred_ema, tf.float32))
with tf.name_scope('summary'):
with tf.name_scope('discriminator'):
tf.summary.scalar('loss_discriminator', loss_dis, ['dis'])
tf.summary.scalar('kl_loss', j_loss, ['dis'])
with tf.name_scope('generator'):
tf.summary.scalar('loss_generator', loss_gen, ['gen'])
with tf.name_scope('images'):
tf.summary.image('gen_images', gen_inp, 10, ['image'])
with tf.name_scope('epoch'):
tf.summary.scalar('accuracy_train', acc_train_pl, ['epoch'])
tf.summary.scalar('accuracy_test_moving_average', acc_test_pl_ema,
['epoch'])
tf.summary.scalar('accuracy_test_raw', acc_test_pl, ['epoch'])
tf.summary.scalar('learning_rate', lr_pl, ['epoch'])
tf.summary.scalar('j_weight', kl_weight, ['epoch'])
sum_op_dis = tf.summary.merge_all('dis')
sum_op_gen = tf.summary.merge_all('gen')
sum_op_im = tf.summary.merge_all('image')
sum_op_epoch = tf.summary.merge_all('epoch')
# training global varialble
global_epoch = tf.Variable(0, trainable=False, name='global_epoch')
global_step = tf.Variable(0, trainable=False, name='global_step')
inc_global_step = tf.assign(global_step, global_step + 1)
inc_global_epoch = tf.assign(global_epoch, global_epoch + 1)
# op initializer for session manager
init_gen = [var.initializer for var in gvars][:-3]
with tf.control_dependencies(init_gen):
op = tf.global_variables_initializer()
init_feed_dict = {
inp: trainx_unl[:FLAGS.batch_size],
unl: trainx_unl[:FLAGS.batch_size],
is_training_pl: True,
kl_weight: 0
}
sv = tf.train.Supervisor(logdir=FLAGS.logdir,
global_step=global_epoch,
summary_op=None,
save_model_secs=0,
init_op=op,
init_feed_dict=init_feed_dict)
inception_scores = []
'''//////training //////'''
print('start training')
with sv.managed_session() as sess:
tf.set_random_seed(rng.randint(2**10))
print('\ninitialization done')
print('Starting training from epoch :%d, step:%d \n' %
(sess.run(global_epoch), sess.run(global_step)))
writer = tf.summary.FileWriter(FLAGS.logdir, sess.graph)
while not sv.should_stop():
epoch = sess.run(global_epoch)
train_batch = sess.run(global_step)
if (epoch >= FLAGS.epoch):
print("Training done")
sv.stop()
break
begin = time.time()
train_loss_lab = train_loss_unl = train_loss_gen = train_acc = test_acc = test_acc_ma = train_j_loss = 0
lr = FLAGS.learning_rate * linear_decay(FLAGS.decay_start,
FLAGS.epoch, epoch)
klw = FLAGS.nabla_w
# construct randomly permuted batches
trainx = []
trainy = []
for t in range(
int(np.ceil(
trainx_unl.shape[0] /
float(txs.shape[0])))): # same size lbl and unlb
inds = rng.permutation(txs.shape[0])
trainx.append(txs[inds])
trainy.append(tys[inds])
trainx = np.concatenate(trainx, axis=0)
trainy = np.concatenate(trainy, axis=0)
trainx_unl = trainx_unl[rng.permutation(
trainx_unl.shape[0])] # shuffling unl dataset
trainx_unl2 = trainx_unl2[rng.permutation(trainx_unl2.shape[0])]
# training
for t in range(nr_batches_train):
display_progression_epoch(t, nr_batches_train)
ran_from = t * FLAGS.batch_size
ran_to = (t + 1) * FLAGS.batch_size
# train discriminator
feed_dict = {
unl: trainx_unl[ran_from:ran_to],
is_training_pl: True,
inp: trainx[ran_from:ran_to],
lbl: trainy[ran_from:ran_to],
lr_pl: lr,
kl_weight: klw
}
_, acc, lu, lb, jl, sm = sess.run([
train_dis_op, accuracy_classifier, loss_lab, loss_unl,
j_loss, sum_op_dis
],
feed_dict=feed_dict)
train_loss_unl += lu
train_loss_lab += lb
train_acc += acc
train_j_loss += jl
if (train_batch % FLAGS.step_print) == 0:
writer.add_summary(sm, train_batch)
# train generator
_, lg, sm = sess.run(
[train_gen_op, loss_gen, sum_op_gen],
feed_dict={
unl: trainx_unl2[ran_from:ran_to],
is_training_pl: True,
lr_pl: lr,
kl_weight: klw
})
train_loss_gen += lg
if (train_batch % FLAGS.step_print) == 0:
writer.add_summary(sm, train_batch)
if (train_batch % FLAGS.freq_print == 0) & (train_batch != 0):
ran_from = np.random.randint(
0, trainx_unl.shape[0] - FLAGS.batch_size)
ran_to = ran_from + FLAGS.batch_size
sm = sess.run(sum_op_im,
feed_dict={
is_training_pl: True,
unl: trainx_unl[ran_from:ran_to]
})
writer.add_summary(sm, train_batch)
train_batch += 1
sess.run(inc_global_step)
train_loss_lab /= nr_batches_train
train_loss_unl /= nr_batches_train
train_loss_gen /= nr_batches_train
train_acc /= nr_batches_train
train_j_loss /= nr_batches_train
# Testing moving averaged model and raw model
if (epoch % FLAGS.freq_test == 0) | (epoch == FLAGS.epoch - 1):
for t in range(nr_batches_test):
ran_from = t * FLAGS.batch_size
ran_to = (t + 1) * FLAGS.batch_size
feed_dict = {
inp: testx[ran_from:ran_to],
lbl: testy[ran_from:ran_to],
is_training_pl: False
}
acc, acc_ema = sess.run(
[accuracy_classifier, accuracy_ema],
feed_dict=feed_dict)
test_acc += acc
test_acc_ma += acc_ema
test_acc /= nr_batches_test
test_acc_ma /= nr_batches_test
sum = sess.run(sum_op_epoch,
feed_dict={
acc_train_pl: train_acc,
acc_test_pl: test_acc,
acc_test_pl_ema: test_acc_ma,
lr_pl: lr,
kl_weight: klw
})
writer.add_summary(sum, epoch)
print(
"Epoch %d | time = %ds | loss gen = %.4f | loss lab = %.4f | loss unl = %.4f "
"| train acc = %.4f| test acc = %.4f | test acc ema = %0.4f"
% (epoch, time.time() - begin, train_loss_gen,
train_loss_lab, train_loss_unl, train_acc, test_acc,
test_acc_ma))
sess.run(inc_global_epoch)
# save snap shot of model
if ((epoch % FLAGS.freq_save == 0) &
(epoch != 0)) | (epoch == FLAGS.epoch - 1):
string = 'model-' + str(epoch)
save_path = os.path.join(FLAGS.logdir, string)
sv.saver.save(sess, save_path)
print("Model saved in file: %s" % (save_path))
print("saving images...")
sample_images = sess.run(gen_inp,
feed_dict={is_training_pl: False})
save_images(sample_images,
os.path.join(FLAGS.logdir, '{:06d}.png'.format(epoch)))
print('images saved @ ' +
os.path.join(FLAGS.logdir, '{:06d}.png'.format(epoch)))
num_images_to_eval = 50000
eval_images = []
num_batches = num_images_to_eval // FLAGS.batch_size + 1
print("Calculating Inception Score. Sampling {} images...".format(
num_images_to_eval))
np.random.seed(0)
for _ in range(num_batches):
images = sess.run(gen_inp, feed_dict={is_training_pl: False})
eval_images.append(images)
np.random.seed()
eval_images = np.vstack(eval_images)
eval_images = eval_images[:num_images_to_eval]
eval_images = np.clip((eval_images + 1.0) * 127.5, 0.0,
255.0).astype(np.uint8)
# Calc Inception score
eval_images = list(eval_images)
inception_score_mean, inception_score_std = get_inception_score(
eval_images)
print("Inception Score: Mean = {} \tStd = {}.".format(
inception_score_mean, inception_score_std))
inception_scores.append(
dict(mean=inception_score_mean, std=inception_score_std))
with open(INCEPTION_FILENAME, 'wb') as f:
pickle.dump(inception_scores, f)
