def train(self, report_kepchs=64):
print('report_kepochs = %d' % report_kepchs)
batch_size = FLAGS.batch_size
with tf.Graph().as_default():
global_step = tf.train.get_or_create_global_step()
ops = self.model(**self.params)
summary_hook = tf.train.SummarySaverHook(
save_steps=(report_kepchs << 10) // batch_size,
output_dir=self.summary_dir,
summary_op=tf.summary.merge_all())
report_hook = utils.HookReport(report_kepchs << 10, batch_size)
stop_hook = tf.train.StopAtStepHook(
last_step=1 + (FLAGS.total_kepochs << 10) // batch_size)
with tf.train.MonitoredTrainingSession(
checkpoint_dir=self.checkpoint_dir,
hooks=[stop_hook],
chief_only_hooks=[summary_hook, report_hook],
save_checkpoint_secs=600,
save_summaries_steps=0) as sess:
self.sess = sess
self.cur_epochs = self.tf_sess.run(global_step) * batch_size
while not sess.should_stop():
self.train_step(ops)
self.cur_epochs = batch_size * self.tf_sess.run(
global_step)
def train(self, report_kimg=1 << 6):
with tf.Graph().as_default():
batch_size = FLAGS.batch
data_in = self.train_data.make_one_shot_iterator().get_next()
global_step = tf.train.get_or_create_global_step()
self.latent_accuracy = self.add_summary_var('latent_accuracy')
self.mean_smoothness = self.add_summary_var('mean_smoothness')
self.mean_distance = self.add_summary_var('mean_distance')
some_float = tf.placeholder(tf.float32, [], 'some_float')
update_summary_var = lambda x: tf.assign(x, some_float)
latent_accuracy_op = update_summary_var(self.latent_accuracy)
mean_smoothness_op = update_summary_var(self.mean_smoothness)
mean_distance_op = update_summary_var(self.mean_distance)
ops = self.model(**self.params)
summary_hook = tf.train.SummarySaverHook(
save_steps=(report_kimg << 10) // batch_size,
output_dir=self.summary_dir,
summary_op=tf.summary.merge_all())
stop_hook = tf.train.StopAtStepHook(
last_step=1 + (FLAGS.total_kimg << 10) // batch_size)
report_hook = utils.HookReport(report_kimg << 10, batch_size)
run_op = lambda op, value: self.tf_sess.run(
op, feed_dict={some_float: value})
with tf.train.MonitoredTrainingSession(
checkpoint_dir=self.checkpoint_dir,
hooks=[stop_hook],
chief_only_hooks=[report_hook, summary_hook],
save_checkpoint_secs=600,
save_summaries_steps=0) as sess:
self.sess = sess
self.cur_nimg = batch_size * self.tf_sess.run(global_step)
while not sess.should_stop():
self.train_step(data_in, ops)
self.cur_nimg = batch_size * self.tf_sess.run(global_step)
def train(self, report_kimg=1 << 6, summary_kimg=1<<6):
batch_size = FLAGS.batch
with tf.Graph().as_default():
data_in = self.train_data.make_one_shot_iterator().get_next()
global_step = tf.train.get_or_create_global_step()
self.latent_accuracy = self.add_summary_var('latent_accuracy')
self.gap_accuracy=self.add_summary_var('gap_accuracy')
self.mean_smoothness = self.add_summary_var('mean_smoothness')
self.mean_distance = self.add_summary_var('mean_distance')
some_float = tf.placeholder(tf.float32, [], 'some_float')
update_summary_var = lambda x: tf.assign(x, some_float)
latent_accuracy_op = update_summary_var(self.latent_accuracy)
gap_accuracy_op = update_summary_var(self.gap_accuracy)
mean_smoothness_op = update_summary_var(self.mean_smoothness)
mean_distance_op = update_summary_var(self.mean_distance)
ops = self.model(**self.params)
summary_hook = tf.train.SummarySaverHook(
save_steps=(summary_kimg << 10) // batch_size,
output_dir=self.summary_dir,
summary_op=tf.summary.merge_all())
stop_hook = tf.train.StopAtStepHook(last_step=1 + (FLAGS.total_kimg << 10) // batch_size)
report_hook = utils.HookReport(report_kimg << 10, batch_size)
run_op = lambda op, value: self.tf_sess.run(op, feed_dict={some_float: value})
with tf.train.MonitoredTrainingSession(
checkpoint_dir=self.checkpoint_dir,
hooks=[stop_hook],
chief_only_hooks=[report_hook, summary_hook],
save_checkpoint_secs=600,
save_summaries_steps=0) as sess:
self.sess = sess
self.cur_nimg = batch_size * self.tf_sess.run(global_step)
while not sess.should_stop():
self.train_step(data_in, ops)
self.cur_nimg = batch_size * self.tf_sess.run(global_step)
if FLAGS.dataset not in ('stl10_unlabeled', "celeba32"):
if self.cur_nimg % (report_kimg << 10) == 0:
if not FLAGS.dataset.startswith("celeba"):
accuracy = self.eval_latent_accuracy(ops)
#gap_accuracy=self.eval_gap_accuracy(ops)
run_op(latent_accuracy_op, accuracy)
#run_op(gap_accuracy_op, gap_accuracy)
if FLAGS.dataset in ('lines32', 'linesym32'):
mean_ds = self.eval_custom_lines32(ops)
run_op(mean_distance_op, mean_ds[0])
run_op(mean_smoothness_op, mean_ds[1])
elif FLAGS.dataset == 'lines32_vertical':
mean_ds = self.eval_custom_lines32_vertical(ops)
run_op(mean_distance_op, mean_ds[0])
run_op(mean_smoothness_op, mean_ds[1])
elif FLAGS.dataset == 'lines32':
mean_ds = self.eval_custom_lines32(ops)
run_op(mean_distance_op, mean_ds[0])
run_op(mean_smoothness_op, mean_ds[1])
def train(self,
report_kimg=1 << 6,
summary_kimg=1 << 6,
eval_inception_kimg=1500):
batch_size = FLAGS.batch
with tf.Graph().as_default():
FLAGS.total_kimg = FLAGS.train_int_kimg + FLAGS.total_kimg
data_in = self.train_data.make_one_shot_iterator().get_next()
global_step = tf.train.get_or_create_global_step()
self.latent_accuracy = self.add_summary_var('latent_accuracy')
self.mean_smoothness = self.add_summary_var('mean_smoothness')
self.mean_distance = self.add_summary_var('mean_distance')
some_float = tf.placeholder(tf.float32, [], 'some_float')
update_summary_var = lambda x: tf.assign(x, some_float)
latent_accuracy_op = update_summary_var(self.latent_accuracy)
ops = self.model(**self.params)
summary_hook = tf.train.SummarySaverHook(
save_steps=(summary_kimg << 10) // batch_size,
output_dir=self.summary_dir,
summary_op=tf.summary.merge_all())
stop_hook = tf.train.StopAtStepHook(
last_step=1 + (FLAGS.total_kimg << 10) // batch_size)
report_hook = utils.HookReport(report_kimg << 10, batch_size)
run_op = lambda op, value: self.tf_sess.run(
op, feed_dict={some_float: value})
with tf.train.MonitoredTrainingSession(
checkpoint_dir=self.checkpoint_dir,
hooks=[stop_hook],
chief_only_hooks=[report_hook, summary_hook],
save_checkpoint_secs=600,
save_summaries_steps=0) as sess:
self.sess = sess
self.cur_nimg = batch_size * self.tf_sess.run(global_step)
while not sess.should_stop():
if self.cur_nimg < (FLAGS.train_int_kimg << 10):
self.train_int_step(data_in, ops)
self.cur_nimg = batch_size * self.tf_sess.run(
global_step)
else:
self.train_step(data_in, ops)
self.cur_nimg = batch_size * self.tf_sess.run(
global_step)
if self.cur_nimg % (report_kimg << 10) == 0:
if self.cur_nimg < (FLAGS.train_int_kimg << 10):
print('TRAIN INTERPOLATION ING...')
accuracy = self.eval_latent_accuracy(ops)
run_op(latent_accuracy_op, accuracy)
if self.cur_nimg % (eval_inception_kimg << 10) == 0:
inc_score = self.eval_inception_and_fid_score(
ops, cur_image=self.cur_nimg)
print('In %d step, inc_score = %.4f' %
(self.cur_nimg, inc_score))
def train(self, report_kimg=1 << 6):
batch_size = FLAGS.batch
with tf.Graph().as_default():
data_in = self.train_data.make_one_shot_iterator().get_next()
global_step = tf.train.get_or_create_global_step()
self.test_accuracy = self.add_summary_var('test_accuracy')
self.train_accuracy = self.add_summary_var('train_accuracy')
some_float = tf.placeholder(tf.float32, [], 'some_float')
update_summary_var = lambda x: tf.assign(x, some_float)
test_accuracy_op = update_summary_var(self.test_accuracy)
train_accuracy_op = update_summary_var(self.train_accuracy)
ops = self.model(**self.params)
summary_hook = tf.train.SummarySaverHook(
save_steps=(report_kimg << 10) // batch_size,
output_dir=self.summary_dir,
summary_op=tf.summary.merge_all())
stop_hook = tf.train.StopAtStepHook(
last_step=1 + (FLAGS.total_kimg << 10) // batch_size)
report_hook = utils.HookReport(report_kimg << 10, batch_size)
run_op = lambda op, value: self.tf_sess.run(
op, feed_dict={some_float: value})
with tf.train.MonitoredTrainingSession(
master=FLAGS.master,
is_chief=(FLAGS.task == 0),
checkpoint_dir=self.checkpoint_dir,
hooks=[stop_hook],
chief_only_hooks=[report_hook, summary_hook],
save_checkpoint_secs=600,
save_summaries_steps=0) as sess:
self.sess = sess
self.cur_nimg = batch_size * self.tf_sess.run(global_step)
while not sess.should_stop():
self.train_step(data_in, ops)
self.cur_nimg = batch_size * self.tf_sess.run(global_step)
if (self.cur_nimg % (report_kimg << 10)
== 0) and FLAGS.dataset == 'mnist32':
accuracy = self.eval_accuracy(ops, self.test_data,
'test')
run_op(test_accuracy_op, accuracy)
accuracy = self.eval_accuracy(ops,
self.train_data,
'train',
batches=200)
run_op(train_accuracy_op, accuracy)
def train(self, report_kimg=1 << 6):
"""
:param report_kimg: report at every report_kimg
:return:
"""
batch_size = FLAGS.batch
with tf.Graph().as_default():
data_in = self.train_data.make_one_shot_iterator().get_next()
global_step = tf.train.get_or_create_global_step()
self.latent_accuracy = self.add_summary_var('latent_accuracy')
self.mean_smoothness = self.add_summary_var('mean_smoothness')
self.mean_distance = self.add_summary_var('mean_distance')
some_float = tf.placeholder(tf.float32, [], 'some_float')
# create initialize op for variable:latent_accuracy,mean_smoothness,mean_distance
update_summary_var = lambda x: tf.assign(x, some_float)
latent_accuracy_op = update_summary_var(self.latent_accuracy)
mean_smoothness_op = update_summary_var(self.mean_smoothness)
mean_distance_op = update_summary_var(self.mean_distance)
# main op
ops = self.model(**self.params)
summary_hook = tf.train.SummarySaverHook(
save_steps=(report_kimg << 9) //
batch_size, # save every steps
output_dir=self.summary_dir,
summary_op=tf.summary.merge_all())
stop_hook = tf.train.StopAtStepHook(
last_step=1 + (FLAGS.total_kimg << 10) // batch_size)
report_hook = utils.HookReport(report_kimg << 10, batch_size)
run_op = lambda op, value: self.tf_sess.run(
op, feed_dict={some_float: value})
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.train.MonitoredTrainingSession(
checkpoint_dir=self.
checkpoint_dir, # automatically restore from ckpt
hooks=[stop_hook],
chief_only_hooks=[
report_hook, summary_hook
], # the hooks are only valid for chief session
save_checkpoint_secs=1000, # every 6 minutes save ckpt
save_summaries_steps=0, # every steps to save summary
config=config) as sess:
self.sess = sess
self.cur_nimg = batch_size * self.tf_sess.run(global_step)
while not sess.should_stop():
# run data_in ops first and then run ops.train_op
self.train_step(data_in, ops)
# MonitoredTrainingSession has an underlying session object: tf_session
# current computed number of image
self.cur_nimg = batch_size * self.tf_sess.run(global_step)
# Time to evaluate classification accuracy
if self.cur_nimg % (report_kimg << 10) == 0:
# return with float accuracy
accuracy = self.eval_latent_accuracy(ops)
# print('eval accuracy:', accuracy, self.cur_nimg)
self.tf_sess.run(latent_accuracy_op,
feed_dict={some_float: accuracy})
if FLAGS.dataset in ('lines32', 'linesym32'):
mean_ds = self.eval_custom_lines32(ops)
run_op(mean_distance_op, mean_ds[0])
run_op(mean_smoothness_op, mean_ds[1])
elif FLAGS.dataset == 'lines32_vertical':
mean_ds = self.eval_custom_lines32_vertical(ops)
run_op(mean_distance_op, mean_ds[0])
run_op(mean_smoothness_op, mean_ds[1])
elif FLAGS.dataset == 'lines32':
mean_ds = self.eval_custom_lines32(ops)
run_op(mean_distance_op, mean_ds[0])
run_op(mean_smoothness_op, mean_ds[1])
def train(self, report_kimg=1 << 6):
"""
:param report_kimg: report at every report_kimg
:return:
"""
batchsz = FLAGS.batchsz
if self.train_graph is None:
self.train_graph = tf.Graph()
with self.train_graph.as_default():
# data_in = self.train_data.make_one_shot_iterator().get_next()
global_step = tf.train.get_or_create_global_step()
some_float = tf.placeholder(tf.float32, [], 'some_float')
self.latent_accuracy = self.add_summary_var('latent_accuracy')
update_summary_var = lambda x: tf.assign(x, some_float)
latent_accuracy_op = update_summary_var(self.latent_accuracy)
summary_hook = tf.train.SummarySaverHook(
save_steps=(report_kimg << 9) // batchsz, # save every steps
output_dir=self.summary_dir,
summary_op=tf.summary.merge_all())
stop_hook = tf.train.StopAtStepHook(
last_step=1 + (FLAGS.total_kimg << 10) // batchsz)
report_hook = utils.HookReport(report_kimg << 7, batchsz)
init_my_summary_op = lambda op, value: self.tf_sess.run(
op, feed_dict={some_float: value})
# main op
ops = self.model(**self.params)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.train.MonitoredTrainingSession(
checkpoint_dir=self.
checkpoint_dir, # automatically restore from ckpt
hooks=[stop_hook],
chief_only_hooks=[
report_hook, summary_hook
], # the hooks are only valid for chief session
save_checkpoint_secs=1000, # every 6 minutes save ckpt
save_summaries_steps=
0, # do NOT save summary into checkpoint_dir
config=config) as sess:
self.sess = sess
self.cur_nimg = batchsz * self.tf_sess.run(global_step)
while not sess.should_stop():
# run data_in ops first and then run ops.train_op
spt_x, spt_y, qry_x, qry_y = self.train_data.get_batch(
batchsz, use_episode=True)
sess.run(ops['meta_op'],
feed_dict={
ops['train_spt_x']: spt_x,
ops['train_spt_y']: spt_y,
ops['train_qry_x']: qry_x,
ops['train_qry_y']: qry_y
})
# MonitoredTrainingSession has an underlying session object: tf_session
# current computed number of image
self.cur_nimg = batchsz * self.tf_sess.run(global_step)
# Time to evaluate classification accuracy
if self.cur_nimg % (report_kimg << 8) == 0:
print('eval...')
# return with float accuracy
accuracy = self.eval_latent_accuracy(ops)
# print('eval accuracy:', accuracy, self.cur_nimg)
# self.tf_sess.run(latent_accuracy_op, feed_dict={some_float: accuracy})
init_my_summary_op(latent_accuracy_op, accuracy)
