def train(self, report_kepchs=64): print('report_kepochs = %d' % report_kepchs) batch_size = FLAGS.batch_size with tf.Graph().as_default(): global_step = tf.train.get_or_create_global_step() ops = self.model(**self.params) summary_hook = tf.train.SummarySaverHook( save_steps=(report_kepchs << 10) // batch_size, output_dir=self.summary_dir, summary_op=tf.summary.merge_all()) report_hook = utils.HookReport(report_kepchs << 10, batch_size) stop_hook = tf.train.StopAtStepHook( last_step=1 + (FLAGS.total_kepochs << 10) // batch_size) with tf.train.MonitoredTrainingSession( checkpoint_dir=self.checkpoint_dir, hooks=[stop_hook], chief_only_hooks=[summary_hook, report_hook], save_checkpoint_secs=600, save_summaries_steps=0) as sess: self.sess = sess self.cur_epochs = self.tf_sess.run(global_step) * batch_size while not sess.should_stop(): self.train_step(ops) self.cur_epochs = batch_size * self.tf_sess.run( global_step)
def train(self, report_kimg=1 << 6): with tf.Graph().as_default(): batch_size = FLAGS.batch data_in = self.train_data.make_one_shot_iterator().get_next() global_step = tf.train.get_or_create_global_step() self.latent_accuracy = self.add_summary_var('latent_accuracy') self.mean_smoothness = self.add_summary_var('mean_smoothness') self.mean_distance = self.add_summary_var('mean_distance') some_float = tf.placeholder(tf.float32, [], 'some_float') update_summary_var = lambda x: tf.assign(x, some_float) latent_accuracy_op = update_summary_var(self.latent_accuracy) mean_smoothness_op = update_summary_var(self.mean_smoothness) mean_distance_op = update_summary_var(self.mean_distance) ops = self.model(**self.params) summary_hook = tf.train.SummarySaverHook( save_steps=(report_kimg << 10) // batch_size, output_dir=self.summary_dir, summary_op=tf.summary.merge_all()) stop_hook = tf.train.StopAtStepHook( last_step=1 + (FLAGS.total_kimg << 10) // batch_size) report_hook = utils.HookReport(report_kimg << 10, batch_size) run_op = lambda op, value: self.tf_sess.run( op, feed_dict={some_float: value}) with tf.train.MonitoredTrainingSession( checkpoint_dir=self.checkpoint_dir, hooks=[stop_hook], chief_only_hooks=[report_hook, summary_hook], save_checkpoint_secs=600, save_summaries_steps=0) as sess: self.sess = sess self.cur_nimg = batch_size * self.tf_sess.run(global_step) while not sess.should_stop(): self.train_step(data_in, ops) self.cur_nimg = batch_size * self.tf_sess.run(global_step)
def train(self, report_kimg=1 << 6, summary_kimg=1<<6): batch_size = FLAGS.batch with tf.Graph().as_default(): data_in = self.train_data.make_one_shot_iterator().get_next() global_step = tf.train.get_or_create_global_step() self.latent_accuracy = self.add_summary_var('latent_accuracy') self.gap_accuracy=self.add_summary_var('gap_accuracy') self.mean_smoothness = self.add_summary_var('mean_smoothness') self.mean_distance = self.add_summary_var('mean_distance') some_float = tf.placeholder(tf.float32, [], 'some_float') update_summary_var = lambda x: tf.assign(x, some_float) latent_accuracy_op = update_summary_var(self.latent_accuracy) gap_accuracy_op = update_summary_var(self.gap_accuracy) mean_smoothness_op = update_summary_var(self.mean_smoothness) mean_distance_op = update_summary_var(self.mean_distance) ops = self.model(**self.params) summary_hook = tf.train.SummarySaverHook( save_steps=(summary_kimg << 10) // batch_size, output_dir=self.summary_dir, summary_op=tf.summary.merge_all()) stop_hook = tf.train.StopAtStepHook(last_step=1 + (FLAGS.total_kimg << 10) // batch_size) report_hook = utils.HookReport(report_kimg << 10, batch_size) run_op = lambda op, value: self.tf_sess.run(op, feed_dict={some_float: value}) with tf.train.MonitoredTrainingSession( checkpoint_dir=self.checkpoint_dir, hooks=[stop_hook], chief_only_hooks=[report_hook, summary_hook], save_checkpoint_secs=600, save_summaries_steps=0) as sess: self.sess = sess self.cur_nimg = batch_size * self.tf_sess.run(global_step) while not sess.should_stop(): self.train_step(data_in, ops) self.cur_nimg = batch_size * self.tf_sess.run(global_step) if FLAGS.dataset not in ('stl10_unlabeled', "celeba32"): if self.cur_nimg % (report_kimg << 10) == 0: if not FLAGS.dataset.startswith("celeba"): accuracy = self.eval_latent_accuracy(ops) #gap_accuracy=self.eval_gap_accuracy(ops) run_op(latent_accuracy_op, accuracy) #run_op(gap_accuracy_op, gap_accuracy) if FLAGS.dataset in ('lines32', 'linesym32'): mean_ds = self.eval_custom_lines32(ops) run_op(mean_distance_op, mean_ds[0]) run_op(mean_smoothness_op, mean_ds[1]) elif FLAGS.dataset == 'lines32_vertical': mean_ds = self.eval_custom_lines32_vertical(ops) run_op(mean_distance_op, mean_ds[0]) run_op(mean_smoothness_op, mean_ds[1]) elif FLAGS.dataset == 'lines32': mean_ds = self.eval_custom_lines32(ops) run_op(mean_distance_op, mean_ds[0]) run_op(mean_smoothness_op, mean_ds[1])
def train(self, report_kimg=1 << 6, summary_kimg=1 << 6, eval_inception_kimg=1500): batch_size = FLAGS.batch with tf.Graph().as_default(): FLAGS.total_kimg = FLAGS.train_int_kimg + FLAGS.total_kimg data_in = self.train_data.make_one_shot_iterator().get_next() global_step = tf.train.get_or_create_global_step() self.latent_accuracy = self.add_summary_var('latent_accuracy') self.mean_smoothness = self.add_summary_var('mean_smoothness') self.mean_distance = self.add_summary_var('mean_distance') some_float = tf.placeholder(tf.float32, [], 'some_float') update_summary_var = lambda x: tf.assign(x, some_float) latent_accuracy_op = update_summary_var(self.latent_accuracy) ops = self.model(**self.params) summary_hook = tf.train.SummarySaverHook( save_steps=(summary_kimg << 10) // batch_size, output_dir=self.summary_dir, summary_op=tf.summary.merge_all()) stop_hook = tf.train.StopAtStepHook( last_step=1 + (FLAGS.total_kimg << 10) // batch_size) report_hook = utils.HookReport(report_kimg << 10, batch_size) run_op = lambda op, value: self.tf_sess.run( op, feed_dict={some_float: value}) with tf.train.MonitoredTrainingSession( checkpoint_dir=self.checkpoint_dir, hooks=[stop_hook], chief_only_hooks=[report_hook, summary_hook], save_checkpoint_secs=600, save_summaries_steps=0) as sess: self.sess = sess self.cur_nimg = batch_size * self.tf_sess.run(global_step) while not sess.should_stop(): if self.cur_nimg < (FLAGS.train_int_kimg << 10): self.train_int_step(data_in, ops) self.cur_nimg = batch_size * self.tf_sess.run( global_step) else: self.train_step(data_in, ops) self.cur_nimg = batch_size * self.tf_sess.run( global_step) if self.cur_nimg % (report_kimg << 10) == 0: if self.cur_nimg < (FLAGS.train_int_kimg << 10): print('TRAIN INTERPOLATION ING...') accuracy = self.eval_latent_accuracy(ops) run_op(latent_accuracy_op, accuracy) if self.cur_nimg % (eval_inception_kimg << 10) == 0: inc_score = self.eval_inception_and_fid_score( ops, cur_image=self.cur_nimg) print('In %d step, inc_score = %.4f' % (self.cur_nimg, inc_score))
def train(self, report_kimg=1 << 6): batch_size = FLAGS.batch with tf.Graph().as_default(): data_in = self.train_data.make_one_shot_iterator().get_next() global_step = tf.train.get_or_create_global_step() self.test_accuracy = self.add_summary_var('test_accuracy') self.train_accuracy = self.add_summary_var('train_accuracy') some_float = tf.placeholder(tf.float32, [], 'some_float') update_summary_var = lambda x: tf.assign(x, some_float) test_accuracy_op = update_summary_var(self.test_accuracy) train_accuracy_op = update_summary_var(self.train_accuracy) ops = self.model(**self.params) summary_hook = tf.train.SummarySaverHook( save_steps=(report_kimg << 10) // batch_size, output_dir=self.summary_dir, summary_op=tf.summary.merge_all()) stop_hook = tf.train.StopAtStepHook( last_step=1 + (FLAGS.total_kimg << 10) // batch_size) report_hook = utils.HookReport(report_kimg << 10, batch_size) run_op = lambda op, value: self.tf_sess.run( op, feed_dict={some_float: value}) with tf.train.MonitoredTrainingSession( master=FLAGS.master, is_chief=(FLAGS.task == 0), checkpoint_dir=self.checkpoint_dir, hooks=[stop_hook], chief_only_hooks=[report_hook, summary_hook], save_checkpoint_secs=600, save_summaries_steps=0) as sess: self.sess = sess self.cur_nimg = batch_size * self.tf_sess.run(global_step) while not sess.should_stop(): self.train_step(data_in, ops) self.cur_nimg = batch_size * self.tf_sess.run(global_step) if (self.cur_nimg % (report_kimg << 10) == 0) and FLAGS.dataset == 'mnist32': accuracy = self.eval_accuracy(ops, self.test_data, 'test') run_op(test_accuracy_op, accuracy) accuracy = self.eval_accuracy(ops, self.train_data, 'train', batches=200) run_op(train_accuracy_op, accuracy)
def train(self, report_kimg=1 << 6): """ :param report_kimg: report at every report_kimg :return: """ batch_size = FLAGS.batch with tf.Graph().as_default(): data_in = self.train_data.make_one_shot_iterator().get_next() global_step = tf.train.get_or_create_global_step() self.latent_accuracy = self.add_summary_var('latent_accuracy') self.mean_smoothness = self.add_summary_var('mean_smoothness') self.mean_distance = self.add_summary_var('mean_distance') some_float = tf.placeholder(tf.float32, [], 'some_float') # create initialize op for variable:latent_accuracy,mean_smoothness,mean_distance update_summary_var = lambda x: tf.assign(x, some_float) latent_accuracy_op = update_summary_var(self.latent_accuracy) mean_smoothness_op = update_summary_var(self.mean_smoothness) mean_distance_op = update_summary_var(self.mean_distance) # main op ops = self.model(**self.params) summary_hook = tf.train.SummarySaverHook( save_steps=(report_kimg << 9) // batch_size, # save every steps output_dir=self.summary_dir, summary_op=tf.summary.merge_all()) stop_hook = tf.train.StopAtStepHook( last_step=1 + (FLAGS.total_kimg << 10) // batch_size) report_hook = utils.HookReport(report_kimg << 10, batch_size) run_op = lambda op, value: self.tf_sess.run( op, feed_dict={some_float: value}) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.train.MonitoredTrainingSession( checkpoint_dir=self. checkpoint_dir, # automatically restore from ckpt hooks=[stop_hook], chief_only_hooks=[ report_hook, summary_hook ], # the hooks are only valid for chief session save_checkpoint_secs=1000, # every 6 minutes save ckpt save_summaries_steps=0, # every steps to save summary config=config) as sess: self.sess = sess self.cur_nimg = batch_size * self.tf_sess.run(global_step) while not sess.should_stop(): # run data_in ops first and then run ops.train_op self.train_step(data_in, ops) # MonitoredTrainingSession has an underlying session object: tf_session # current computed number of image self.cur_nimg = batch_size * self.tf_sess.run(global_step) # Time to evaluate classification accuracy if self.cur_nimg % (report_kimg << 10) == 0: # return with float accuracy accuracy = self.eval_latent_accuracy(ops) # print('eval accuracy:', accuracy, self.cur_nimg) self.tf_sess.run(latent_accuracy_op, feed_dict={some_float: accuracy}) if FLAGS.dataset in ('lines32', 'linesym32'): mean_ds = self.eval_custom_lines32(ops) run_op(mean_distance_op, mean_ds[0]) run_op(mean_smoothness_op, mean_ds[1]) elif FLAGS.dataset == 'lines32_vertical': mean_ds = self.eval_custom_lines32_vertical(ops) run_op(mean_distance_op, mean_ds[0]) run_op(mean_smoothness_op, mean_ds[1]) elif FLAGS.dataset == 'lines32': mean_ds = self.eval_custom_lines32(ops) run_op(mean_distance_op, mean_ds[0]) run_op(mean_smoothness_op, mean_ds[1])
def train(self, report_kimg=1 << 6): """ :param report_kimg: report at every report_kimg :return: """ batchsz = FLAGS.batchsz if self.train_graph is None: self.train_graph = tf.Graph() with self.train_graph.as_default(): # data_in = self.train_data.make_one_shot_iterator().get_next() global_step = tf.train.get_or_create_global_step() some_float = tf.placeholder(tf.float32, [], 'some_float') self.latent_accuracy = self.add_summary_var('latent_accuracy') update_summary_var = lambda x: tf.assign(x, some_float) latent_accuracy_op = update_summary_var(self.latent_accuracy) summary_hook = tf.train.SummarySaverHook( save_steps=(report_kimg << 9) // batchsz, # save every steps output_dir=self.summary_dir, summary_op=tf.summary.merge_all()) stop_hook = tf.train.StopAtStepHook( last_step=1 + (FLAGS.total_kimg << 10) // batchsz) report_hook = utils.HookReport(report_kimg << 7, batchsz) init_my_summary_op = lambda op, value: self.tf_sess.run( op, feed_dict={some_float: value}) # main op ops = self.model(**self.params) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.train.MonitoredTrainingSession( checkpoint_dir=self. checkpoint_dir, # automatically restore from ckpt hooks=[stop_hook], chief_only_hooks=[ report_hook, summary_hook ], # the hooks are only valid for chief session save_checkpoint_secs=1000, # every 6 minutes save ckpt save_summaries_steps= 0, # do NOT save summary into checkpoint_dir config=config) as sess: self.sess = sess self.cur_nimg = batchsz * self.tf_sess.run(global_step) while not sess.should_stop(): # run data_in ops first and then run ops.train_op spt_x, spt_y, qry_x, qry_y = self.train_data.get_batch( batchsz, use_episode=True) sess.run(ops['meta_op'], feed_dict={ ops['train_spt_x']: spt_x, ops['train_spt_y']: spt_y, ops['train_qry_x']: qry_x, ops['train_qry_y']: qry_y }) # MonitoredTrainingSession has an underlying session object: tf_session # current computed number of image self.cur_nimg = batchsz * self.tf_sess.run(global_step) # Time to evaluate classification accuracy if self.cur_nimg % (report_kimg << 8) == 0: print('eval...') # return with float accuracy accuracy = self.eval_latent_accuracy(ops) # print('eval accuracy:', accuracy, self.cur_nimg) # self.tf_sess.run(latent_accuracy_op, feed_dict={some_float: accuracy}) init_my_summary_op(latent_accuracy_op, accuracy)