def train(infer_z, noisy_y, C):
"""Train Clothing1M for a number of steps."""
with tf.Graph().as_default():
global_step = tf.train.get_or_create_global_step()
# Get images and labels for Clothing1M.
# Force input pipeline to CPU:0 to avoid operations sometimes ending up on
# GPU and resulting in a slow down.
with tf.device('/cpu:0'):
indices, images, labels = clothing1M.distorted_inputs()
# Build a Graph that computes the logits predictions from the
# inference model.
is_training = tf.placeholder(tf.bool)
logits = clothing1M.inference_resnet_own(images,training=is_training)
preds = tf.nn.softmax(logits)
# approximate Gibbs sampling
T = tf.placeholder(tf.float32,shape=[clothing1M.NUM_CLASSES+1,clothing1M.NUM_CLASSES],name='transition')
unnorm_probs = preds * tf.gather(tf.transpose(T,[1,0]),labels)
probs = unnorm_probs / tf.reduce_sum(unnorm_probs,axis=1,keepdims=True)
sampler = OneHotCategorical(probs=probs)
labels_ = tf.stop_gradient(tf.argmax(sampler.sample(),axis=1))
# Calculate loss.
loss = clothing1M.loss(logits, labels_)
# Build a Graph that trains the model with one batch of examples and
# updates the model parameters.
train_op = clothing1M.train(loss, global_step)
# Calculate prediction
# acc_op contains acc and update_op. So it is the cumulative accuracy when sess runs acc_op
# if you only want to inspect acc of each batch, just sess run acc_op[0]
acc_op = tf.metrics.accuracy(labels, tf.argmax(logits,axis=1))
tf.summary.scalar('training accuracy', acc_op[0])
#### build scalffold for MonitoredTrainingSession to restore the variables you wish
variables_to_restore = []
#variables_to_restore += [var for var in tf.trainable_variables() if ('dense' not in var.name and 'logits_T' not in var.name)]
variables_to_restore += tf.trainable_variables()
variables_to_restore += [g for g in tf.global_variables() if 'moving_mean' in g.name or 'moving_variance' in g.name]
for var in variables_to_restore:
print(var.name)
ckpt = tf.train.get_checkpoint_state(FLAGS.init_dir)
init_assign_op, init_feed_dict = tf.contrib.framework.assign_from_checkpoint(
ckpt.model_checkpoint_path, variables_to_restore)
def InitAssignFn(scaffold,sess):
sess.run(init_assign_op, init_feed_dict)
scaffold = tf.train.Scaffold(saver=tf.train.Saver(), init_fn=InitAssignFn)
class _LoggerHook(tf.train.SessionRunHook):
"""Logs loss and runtime."""
def begin(self):
self._step = -1
self._start_time = time.time()
def before_run(self, run_context):
self._step += 1
return tf.train.SessionRunArgs(tf.get_collection('losses')[0]) # Asks for loss value.
def after_run(self, run_context, run_values):
if self._step % FLAGS.log_frequency == 0:
current_time = time.time()
duration = current_time - self._start_time
self._start_time = current_time
loss_value = run_values.results
examples_per_sec = FLAGS.log_frequency * FLAGS.batch_size / duration
sec_per_batch = float(duration / FLAGS.log_frequency)
format_str = ('%s: step %d, loss = %.6f (%.1f examples/sec; %.3f '
'sec/batch)')
print (format_str % (datetime.now(), self._step, loss_value,
examples_per_sec, sec_per_batch))
max_steps = int(math.ceil(clothing1M.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN*FLAGS.num_epochs/FLAGS.batch_size))
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)
with tf.train.MonitoredTrainingSession(
checkpoint_dir=FLAGS.train_dir,
scaffold = scaffold,
hooks=[tf.train.StopAtStepHook(last_step=max_steps),
tf.train.NanTensorHook(loss),
_LoggerHook()],
save_checkpoint_secs=60,
config=tf.ConfigProto(
log_device_placement=FLAGS.log_device_placement, gpu_options=gpu_options)) as mon_sess:
alpha = 1.0
C_init = C.copy()
trans_init = (C_init + alpha) / np.sum(C_init + alpha, axis=1, keepdims=True)
warming_up_step_1 = 20000
warming_up_step_2 = 40000
step = 0
freq_step = 50000
## warming up transition
with open(FLAGS.transition) as f:
data = pickle.load(f)
trans_warming_1 = np.concatenate([np_smoothing_eye(clothing1M.NUM_CLASSES,delta=0.05),np.ones([1,clothing1M.NUM_CLASSES])*1.0/clothing1M.NUM_CLASSES],axis=0)
trans_warming_2 = data[0].copy()
trans = data[0].copy()
exemplars = []
while not mon_sess.should_stop():
alpha = 1.0
if (step >= warming_up_step_2) and (step%freq_step == 0):
trans = (C + alpha) / np.sum(C + alpha, axis=1, keepdims=True)
if step < warming_up_step_1:
res = mon_sess.run([train_op,acc_op,global_step,indices,labels,labels_],feed_dict={is_training:True, T: trans_warming_1})
elif step < warming_up_step_2:
res = mon_sess.run([train_op,acc_op,global_step,indices,labels,labels_],feed_dict={is_training:True, T: trans_warming_2})
else:
res = mon_sess.run([train_op,acc_op,global_step,indices,labels,labels_],feed_dict={is_training:True, T: trans})
for i in xrange(res[3].shape[0]):
ind = res[3][i]
#print(ind,noisy_y[ind],res[4][i])
assert noisy_y[ind] == res[4][i]
C[infer_z[ind]][res[4][i]] -= 1
assert C[infer_z[ind]][noisy_y[ind]] >= 0
infer_z[ind] = res[5][i]
C[infer_z[ind]][res[4][i]] += 1
#print(res[4][i],res[5][i])
step = res[2]
if step % 1000 == 0:
print('Counting matrix\n', C)
print('Counting matrix\n', C_init)
print('Transition matrix\n', trans)
print('Transition matrix\n', trans_init)
if step % 20000 == 0:
exemplars.append([infer_z.keys(), infer_z.values(), C])
with open('varC_learnt_%s.pkl'%FLAGS.transition[:-4],'w') as w:
pickle.dump(exemplars,w)
def train():
"""Train Clothing1M for a number of steps."""
with tf.Graph().as_default():
global_step = tf.train.get_or_create_global_step()
# Get images and labels for Clothing1M.
# Force input pipeline to CPU:0 to avoid operations sometimes ending up on
# GPU and resulting in a slow down.
with tf.device('/cpu:0'):
indices, images, labels = clothing1M.distorted_inputs()
# Build a Graph that computes the logits predictions from the
# inference model.
is_training = tf.placeholder(tf.bool)
logits = clothing1M.inference_resnet_own(images, training=is_training)
# Calculate loss.
loss = clothing1M.loss(logits, labels)
# Build a Graph that trains the model with one batch of examples and
# updates the model parameters.
train_op = clothing1M.train(loss, global_step)
# Calculate prediction
# acc_op contains acc and update_op. So it is the cumulative accuracy when sess runs acc_op
# if you only want to inspect acc of each batch, just sess run acc_op[0]
acc_op = tf.metrics.accuracy(labels, tf.argmax(logits, axis=1))
tf.summary.scalar('training accuracy', acc_op[0])
#### build scalffold for MonitoredTrainingSession to restore the variables you wish
variables_to_restore = []
variables_to_restore += [
var for var in tf.trainable_variables() if 'dense' not in var.name
]
variables_to_restore += [
g for g in tf.global_variables()
if 'moving_mean' in g.name or 'moving_variance' in g.name
]
for var in variables_to_restore:
print(var.name)
ckpt = tf.train.get_checkpoint_state(FLAGS.init_dir)
init_assign_op, init_feed_dict = tf.contrib.framework.assign_from_checkpoint(
ckpt.model_checkpoint_path, variables_to_restore)
def InitAssignFn(scaffold, sess):
sess.run(init_assign_op, init_feed_dict)
scaffold = tf.train.Scaffold(saver=tf.train.Saver(),
init_fn=InitAssignFn)
class _LoggerHook(tf.train.SessionRunHook):
"""Logs loss and runtime."""
def begin(self):
self._step = -1
self._start_time = time.time()
def before_run(self, run_context):
self._step += 1
return tf.train.SessionRunArgs(
tf.get_collection('losses')[0]) # Asks for loss value.
def after_run(self, run_context, run_values):
if self._step % FLAGS.log_frequency == 0:
current_time = time.time()
duration = current_time - self._start_time
self._start_time = current_time
loss_value = run_values.results
examples_per_sec = FLAGS.log_frequency * FLAGS.batch_size / duration
sec_per_batch = float(duration / FLAGS.log_frequency)
format_str = (
'%s: step %d, loss = %.6f (%.1f examples/sec; %.3f '
'sec/batch)')
print(format_str % (datetime.now(), self._step, loss_value,
examples_per_sec, sec_per_batch))
max_steps = int(
math.ceil(clothing1M.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN *
FLAGS.num_epochs / FLAGS.batch_size))
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)
with tf.train.MonitoredTrainingSession(
checkpoint_dir=FLAGS.train_dir,
scaffold=scaffold,
hooks=[
tf.train.StopAtStepHook(last_step=max_steps),
tf.train.NanTensorHook(loss),
_LoggerHook()
],
save_checkpoint_secs=60,
config=tf.ConfigProto(
log_device_placement=FLAGS.log_device_placement,
gpu_options=gpu_options)) as mon_sess:
while not mon_sess.should_stop():
res = mon_sess.run([train_op, acc_op, global_step],
feed_dict={is_training: True})