def distorted_inputs():
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin')
images, labels = mnist_input.distorted_inputs(data_dir=data_dir,
batch_size=FLAGS.batch_size)
return images, labels
def distorted_inputs():
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin')
images, labels = mnist_input.distorted_inputs(data_dir=data_dir,
batch_size=FLAGS.batch_size)
if FLAGS.use_fp16:
images = tf.cast(images, tf.float16)
labels = tf.cast(labels, tf.float16)
return images, labels
def distorted_inputs():
"""Construct distorted input for MNIST
Returns:
images: Images.
"""
if not FLAGS.data_dir:
raise ValueError('please supply a data_dir')
images, labels = mnist_input.distorted_inputs(data_dir=FLAGS.data_dir,
batch_size=FLAGS.batch_size)
if FLAGS.use_fp16:
images = tf.cast(images, tf.float16)
labels = tf.cast(labels, tf.float16)
return images, labels
"""Directory where to write event logs """
"""and checkpoint.""")
tf.app.flags.DEFINE_integer('max_steps', 5000,
"""Number of batches to run.""")
tf.app.flags.DEFINE_boolean('log_device_placement', False,
"""Whether to log device placement.""")
INITIAL_LEARNING_RATE = 0.01
NUM_CLASSES = 10
FT_DIM = 3
CODE_LEN = 20
imHeight = 28
imWidth = 28
numCh = 1
# Get images and labels
images_tr, labels_tr = mnistip.distorted_inputs(randFlip=True)
init = tf.initialize_all_variables()
sess = tf.InteractiveSession()
sess.run(init)
tf.train.start_queue_runners(sess=sess)
_images, _labels = sess.run([images_tr, labels_tr])
reshaped_vis = np.squeeze(_images)
ims("results/base.jpg",merge(reshaped_vis[:64],[8,8]))
print(_images.shape)
def train():
with tf.Graph().as_default():
z_dim = 100
fc_dim = 512 # 1568
################# MODEL + OPTIMIZER
# Get images and labels
images_tr, labels_tr = mnistip.distorted_inputs(randFlip=True)
images_ev, labels_ev = mnistip.inputs(eval_data=True, numPrThread=1)
# build model
#### Placeholders
images = tf.placeholder(
dtype=tf.float32,
shape=[FLAGS.batch_size, imHeight, imWidth, numCh])
#####
zin = tf.placeholder(tf.float32, [FLAGS.batch_size, z_dim], name="z")
# Generator
G = gnm.generator(zin)
# Discriminators
D_prob_real, D_logit_real = gnm.discriminator(images)
with tf.variable_scope(tf.get_variable_scope()) as scope:
pass
D_prob_fake, D_logit_fake = gnm.discriminator(G, reuse=True)
# Losses
dloss_real = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=D_logit_real, labels=tf.ones_like(D_logit_real)))
dloss_fake = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=D_logit_fake, labels=tf.zeros_like(D_logit_fake)))
dloss = dloss_real + dloss_fake
gloss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=D_logit_fake, labels=tf.ones_like(D_logit_fake)))
# Optimizer
t_vars = tf.trainable_variables()
d_vars = [var for var in t_vars if 'd_' in var.name]
g_vars = [var for var in t_vars if 'g_' in var.name]
d_optim = tf.train.AdamOptimizer(learningrate, beta1=beta1)
g_optim = tf.train.AdamOptimizer(learningrate, beta1=beta1)
grads_d = tf.gradients(dloss, d_vars)
grads_g = tf.gradients(gloss, g_vars)
with tf.variable_scope(scope):
train_d = d_optim.apply_gradients(zip(grads_d, d_vars))
train_g = g_optim.apply_gradients(zip(grads_g, g_vars))
##### Tensorflow training
#####################
# Create a saver.
saver = tf.train.Saver()
if (isModelFT):
saver1 = tf.train.Saver(tf.trainable_variables())
# Build an initialization operation to run below.
init = tf.initialize_all_variables()
# Start running operations on the Graph.
sess = tf.Session(config=tf.ConfigProto(
log_device_placement=FLAGS.log_device_placement))
sess.run(init)
if (isModelFT):
saver1.restore(sess, restoreFileName)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
############## TRAIN
visualization, ev_labels = sess.run([images_ev, labels_ev])
reshaped_vis = np.squeeze(visualization)
ims("results/real.jpg", merge_gs(reshaped_vis[:49], [7, 7]))
display_z = np.random.uniform(-1, 1, [FLAGS.batch_size, z_dim]).astype(
np.float32)
for epoch in xrange(1):
for steps_ in xrange(200001):
batch_images, _ = sess.run([images_tr, labels_tr])
batch_z = np.random.uniform(
-1, 1, [FLAGS.batch_size, z_dim]).astype(np.float32)
# _, lossD, lossGen = sess.run([train_op, dloss, gloss], feed_dict={images: batch_images, zin: batch_z})
for k in xrange(1):
_, lossD = sess.run([train_d, dloss],
feed_dict={
images: batch_images,
zin: batch_z
})
for k in xrange(1):
_, lossG = sess.run([train_g, gloss],
feed_dict={zin: batch_z})
if steps_ % 100 == 0:
format_str = (
'%s: Step %d, D-LOSS = %.2f, G-loss = %.2f\n')
print(format_str % (datetime.now(), steps_, lossD, lossG))
if steps_ % 200 == 0:
imgGen = sess.run([G], feed_dict={zin: display_z})
imgGen = np.squeeze(imgGen)
ims("results/" + str(steps_) + ".jpg",
merge_gs(imgGen[0:49], [7, 7]))
if steps_ % 1000 == 0:
checkpoint_path = os.path.join(
FLAGS.train_dir, 'model-' + netName + '.ckpt')
saver.save(sess, checkpoint_path, global_step=steps_)
def train():
with tf.Graph().as_default():
global_step = tf.Variable(0, trainable=False)
#### For training CNN
images = tf.placeholder(dtype=tf.float32, shape=[FLAGS.batch_size, imHeight, imWidth, numCh])
labels = tf.placeholder(dtype=tf.int32, shape=[FLAGS.batch_size])
#####
# Get images and labels
images_tr, labels_tr = mnistip.distorted_inputs(randFlip=True)
images_ev, labels_ev = mnistip.inputs(eval_data=True, numPrThread=1)
# images_ev, labels_ev = mnistip.inputs(eval_data=True)
########################
# CL ZONE
# logits_id, local, cent_var, kappaVal, wts, t_num_dim = cvn.inference_cl_cnn(images, CODE_LEN, NUM_CLASSES)
logits_id, local, kappaVal, wts, t_num_dim = cvn.inference_rec_net(images, CODE_LEN, NUM_CLASSES)
# Losses -CL
loss_softmax_id = cvn.loss_softmax(logits_id, labels)
loss_combined = cvn.loss_total()
# Draw new sample
# weight_maps = tf.gather(tf.transpose(wts), labels)
guessed_z = local
# guessed_z = tf.add(cent_var, weight_maps)
# guessed_z = weight_maps
# Losses - Generation
im_gen = cvn.generation(guessed_z, t_num_dim)
# Compute Loss Values
generation_loss = -tf.reduce_sum(images * tf.log(1e-8 + im_gen) + (1-images) * tf.log(1e-8 + 1 - im_gen),[1,2,3])
# total_loss = tf.reduce_mean(generation_loss)*0.001 + loss_combined
# total_loss = tf.reduce_mean(generation_loss) + loss_combined
# total_loss = tf.reduce_mean(loss_combined)
total_loss = tf.reduce_mean(generation_loss)
# Optimize now
# Apply variable specific learning rate for optimization
var_rec = [v for v in tf.trainable_variables() if(v.name.lower().find('rec_') >= 0)]
var_gen = [v for v in tf.trainable_variables() if(v.name.lower().find('gen_') >= 0)]
opt_rec = tf.train.AdamOptimizer(INITIAL_LEARNING_RATE*0.001)
opt_gen = tf.train.AdamOptimizer(INITIAL_LEARNING_RATE)
grads = tf.gradients(total_loss, var_rec + var_gen)
grads_rec = grads[:len(var_rec)]
grads_gen = grads[len(var_rec):]
train_rec = opt_rec.apply_gradients(zip(grads_rec, var_rec))
train_gen = opt_gen.apply_gradients(zip(grads_gen, var_gen))
train_op = tf.group(train_rec, train_gen)
#####################
# Create a saver.
saver = tf.train.Saver()
if(isModelFT):
saver1 = tf.train.Saver(tf.trainable_variables())
# Build an initialization operation to run below.
init = tf.initialize_all_variables()
# Start running operations on the Graph.
sess = tf.Session(config=tf.ConfigProto(
log_device_placement=FLAGS.log_device_placement))
sess.run(init)
if(isModelFT):
saver1.restore(sess, restoreFileName)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
#########################
visualization, ev_labels = sess.run([images_ev, labels_ev])
reshaped_vis = np.squeeze(visualization)
ims("results/base.jpg",merge(reshaped_vis[:64],[8,8]))
#########################
for step in xrange(FLAGS.max_steps):
_images, _labels = sess.run([images_tr, labels_tr])
_, lossSM, lossTot, lossGen = sess.run([train_op, loss_softmax_id, loss_combined, generation_loss], feed_dict={images: _images, labels: _labels})
if step % 100 == 0:
format_str = ('%s: Step %d, GEN-LOSS = %.2f, SM-loss = %.2f, T-loss = %.2f\n')
print (format_str % (datetime.now(), step, np.mean(lossGen), lossSM, lossTot))
# save intermediate results
generated_test = sess.run(im_gen, feed_dict={images: visualization, labels: ev_labels})
generated_test = np.squeeze(generated_test)
ims("results/"+str(step)+'_'+saveImPrefix+".jpg",merge(generated_test[:64],[8,8]))
###########################
# Evaluate test set
###########################
numTestStep = int(mnistip.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL / FLAGS.batch_size)
predictions_id = np.ndarray(shape=(mnistip.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL, mnistip.NUM_CLASSES), dtype = np.float64)
ftVec = np.ndarray(shape=(mnistip.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL,FT_DIM), dtype = np.float64)
tLabels = np.ndarray(shape=(mnistip.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL), dtype = np.float64)
if step % 500 == 0:
# Evaluate
print("====== Evaluating ID classification ========\n")
for step_ev in xrange(numTestStep):
_images, _labels = sess.run([images_ev, labels_ev])
stIndx = step_ev*FLAGS.batch_size
edIndx = (step_ev+1)*FLAGS.batch_size
_fts, tpred_id = sess.run([local, logits_id], feed_dict={images: _images})
predictions_id[stIndx:edIndx, :] = np.asarray(tpred_id)
ftVec[stIndx:edIndx, :] = np.asarray(_fts)
tLabels[stIndx:edIndx] = np.asarray(_labels)
obs_labels = np.argmax(predictions_id, axis=1)
#print(lab.dtype)
sum_ = np.sum(tLabels==obs_labels)
acc_id = sum_/float(mnistip.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL)
print('================================')
format_str = ('%s: Step %d, ID_Acc = %.5f\n')
print (format_str % (datetime.now(), step, acc_id))
print('================================')
if step % 1000 == 0:
checkpoint_path = os.path.join(FLAGS.train_dir, 'model-'+netName+'.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
def train():
with tf.Graph().as_default():
global_step = tf.Variable(0, trainable=False)
#### For training CNN
images = tf.placeholder(
dtype=tf.float32,
shape=[FLAGS.batch_size, imHeight, imWidth, numCh])
labels = tf.placeholder(dtype=tf.int32, shape=[FLAGS.batch_size])
learning_rate = tf.placeholder(dtype=tf.float32, shape=[])
#####
# Get images and labels
images_tr, labels_tr = mnistip.distorted_inputs(randFlip=True)
images_ev, labels_ev = mnistip.inputs(eval_data=True)
# Build a Graph that computes the logits predictions
# local, logits_id = mnist_net.inference_id_classification(images)
logits_id, local, kappaVal = mncnn.inference(images)
# Calculate losses...
# loss_L2 = mnist_net.loss_L2()
#loss_softmax_id = mnist_net.loss_softmax(logits_id, labels)
#loss_combined = mnist_net.loss_combined_no_param()
loss_softmax_id = mncnn.loss_softmax(logits_id, labels)
loss_combined = mncnn.loss_total()
train_op = tf.train.MomentumOptimizer(learning_rate,
0.9).minimize(loss_combined)
# Create a saver.
saver = tf.train.Saver()
if (isModelFT):
saver1 = tf.train.Saver(tf.trainable_variables())
# Build an initialization operation to run below.
init = tf.initialize_all_variables()
# Start running operations on the Graph.
sess = tf.Session(config=tf.ConfigProto(
log_device_placement=FLAGS.log_device_placement))
sess.run(init)
if (isModelFT):
saver1.restore(sess, restoreFileName)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
if (saveLogFile):
if (os.path.isfile(fname)):
os.remove(fname)
f_handle = open(fname, 'a')
# for step in xrange(1):
lr_ = INITIAL_LEARNING_RATE
mulsteplr = [6, 8, 10]
stEpoch = 1
mulsteplr = np.array(mulsteplr)
currEpoch = int(stEpoch * FLAGS.batch_size /
mnistip.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN)
mulstepCnt = np.where(currEpoch < mulsteplr)[0][0]
lr_ = lr_**(mulstepCnt + 1)
for step in xrange(stEpoch, FLAGS.max_steps):
# Learning rate decrease policy
currEpoch = int(step * FLAGS.batch_size /
mnistip.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN)
if (currEpoch >= mulsteplr[mulstepCnt]):
print(str(currEpoch) + ':: Decreasing learning rate')
lr_ = 0.8 * lr_
mulstepCnt = mulstepCnt + 1
_images, _labels = sess.run([images_tr, labels_tr])
_, lossID, lossComb, kappaVal_ = sess.run(
[train_op, loss_softmax_id, loss_combined, kappaVal],
feed_dict={
images: _images,
labels: _labels,
learning_rate: lr_
})
if step % 100 == 0:
format_str = (
'%s: Step %d, LR=%.4f, ID-loss = %.2f, T-loss = %.2f, Kappa = %.2f\n'
)
print(format_str %
(datetime.now(), step, lr_, lossID, lossComb, kappaVal_))
if (saveLogFile):
f_handle.write(format_str % (datetime.now(), step, lr_,
lossID, lossComb, kappaVal_))
assert not np.isnan(lossComb), 'Model diverged with loss = NaN'
# Save the model checkpoint periodically.
if step % 1000 == 0 or (step + 1) == FLAGS.max_steps:
if (isSaveModel):
if isModelFT:
checkpoint_path = os.path.join(
FLAGS.train_dir, 'model-' + netName + '-ft.ckpt')
else:
checkpoint_path = os.path.join(
FLAGS.train_dir, 'model-' + netName + '.ckpt')
print('saving model ...')
saver.save(sess, checkpoint_path, global_step=step)
###########################
# Evaluate test set
###########################
numTestStep = int(mnistip.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL /
FLAGS.batch_size)
predictions_id = np.ndarray(
shape=(mnistip.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL,
mnistip.NUM_CLASSES),
dtype=np.float64)
ftVec = np.ndarray(
shape=(mnistip.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL, FT_DIM),
dtype=np.float64)
tLabels = np.ndarray(
shape=(mnistip.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL),
dtype=np.float64)
# Evaluate
print("====== Evaluating ID classification ========\n")
for step_ev in xrange(numTestStep):
_images, _labels = sess.run([images_ev, labels_ev])
stIndx = step_ev * FLAGS.batch_size
edIndx = (step_ev + 1) * FLAGS.batch_size
_fts, tpred_id = sess.run([local, logits_id],
feed_dict={images: _images})
predictions_id[stIndx:edIndx, :] = np.asarray(tpred_id)
ftVec[stIndx:edIndx, :] = np.asarray(_fts)
tLabels[stIndx:edIndx] = np.asarray(_labels)
obs_labels = np.argmax(predictions_id, axis=1)
#print(lab.dtype)
sum_ = np.sum(tLabels == obs_labels)
acc_id = sum_ / float(mnistip.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL)
print('================================')
format_str = ('%s: Step %d, ID_Acc = %.5f\n')
np.savez(netName + '_' + str(step), X=ftVec, y=tLabels)
print(format_str % (datetime.now(), step, acc_id))
if (saveLogFile):
f_handle.write(
'==================================================\n')
f_handle.write(format_str % (datetime.now(), step, acc_id))
f_handle.write(
'==================================================\n')
def train():
with tf.Graph().as_default():
global_step = tf.Variable(0, trainable=False)
########################
# Get images and labels
images_tr, labels_tr = mnistip.distorted_inputs(randFlip=False)
images_ev, labels_ev = mnistip.inputs(eval_data=True)
########################
# VAE ZONE
images = tf.placeholder(dtype=tf.float32, shape=[FLAGS.batch_size, imHeight, imWidth, numCh])
# vae_code = tf.placeholder(dtype=tf.float32, shape=[FLAGS.batch_size, CODE_LEN])
# Define Encoder
z_mean, z_stddev, t_num_dim = ved.recognition(images, CODE_LEN)
# Draw new sample
samples = tf.random_normal([FLAGS.batch_size,CODE_LEN],0,1,dtype=tf.float32)
guessed_z = z_mean + (z_stddev * samples)
# Define Decoder
im_gen = ved.generation(guessed_z, t_num_dim)
# Compute Loss Values
generation_loss = -tf.reduce_sum(images * tf.log(1e-8 + im_gen) + (1-images) * tf.log(1e-8 + 1 - im_gen),[1,2,3])
latent_loss = 0.5 * tf.reduce_sum(tf.square(z_mean) + tf.square(z_stddev) - tf.log(tf.square(z_stddev)) - 1,1)
total_loss = tf.reduce_mean(generation_loss + latent_loss)
# Optimize now
train_op = tf.train.AdamOptimizer(0.001).minimize(total_loss)
#####################
# Build an initialization operation to run below.
init = tf.initialize_all_variables()
# Start running operations on the Graph.
sess = tf.Session(config=tf.ConfigProto(
log_device_placement=FLAGS.log_device_placement))
sess.run(init)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
#########################
visualization, _ = sess.run([images_ev, labels_ev])
reshaped_vis = np.squeeze(visualization)
ims("results/base.jpg",merge(reshaped_vis[:64],[8,8]))
for step in xrange(FLAGS.max_steps):
_images, _labels = sess.run([images_tr, labels_tr])
_, lossGen, lossLat = sess.run([train_op, generation_loss, latent_loss], feed_dict={images: _images})
if step % 20 == 0:
format_str = ('%s: Step %d, GEN-loss = %.2f, LAT-loss = %.2f\n')
print (format_str % (datetime.now(), step, np.mean(lossGen), np.mean(lossLat)))
# save intermediate results
generated_test = sess.run(im_gen, feed_dict={images: visualization})
generated_test = np.squeeze(generated_test)
ims("results/"+str(step)+".jpg",merge(generated_test[:64],[8,8]))