def main(_): pp.pprint(flags.FLAGS.__flags) cfg_stage_i = config_from_yaml(FLAGS.cfg_stage_I) cfg = config_from_yaml(FLAGS.cfg_stage_II) if not os.path.exists(cfg.CHECKPOINT_DIR): os.makedirs(cfg.CHECKPOINT_DIR) if not os.path.exists(cfg.SAMPLE_DIR): os.makedirs(cfg.SAMPLE_DIR) if not os.path.exists(cfg.LOGS_DIR): os.makedirs(cfg.LOGS_DIR) run_config = tf.ConfigProto() run_config.gpu_options.allow_growth = True datadir = cfg.DATASET_DIR dataset = TextDataset(datadir, 256) filename_test = '%s/test' % datadir dataset.test = dataset.get_data(filename_test) filename_train = '%s/train' % datadir dataset.train = dataset.get_data(filename_train) with tf.Session(config=run_config) as sess: if cfg.EVAL.FLAG: stage_i = ConditionalGanStageI(cfg_stage_i, build_model=False) stage_ii = ConditionalGan(stage_i, cfg, build_model=False) stage_ii_eval = StageIIEval( sess=sess, model=stage_ii, dataset=dataset, cfg=cfg, ) stage_ii_eval.evaluate_inception() elif cfg.TRAIN.FLAG: stage_i = ConditionalGanStageI(cfg_stage_i, build_model=False) stage_ii = ConditionalGan(stage_i, cfg) show_all_variables() stage_ii_trainer = ConditionalGanTrainer( sess=sess, model=stage_ii, dataset=dataset, cfg=cfg, cfg_stage_i=cfg_stage_i, ) stage_ii_trainer.train() else: stage_i = ConditionalGanStageI(cfg_stage_i, build_model=False) stage_ii = ConditionalGan(stage_i, cfg, build_model=False) stage_ii_eval = StageIIVisualizer( sess=sess, model=stage_ii, dataset=dataset, cfg=cfg, ) stage_ii_eval.visualize()
def main(_): pp.pprint(flags.FLAGS.__flags) cfg = config_from_yaml(FLAGS.cfg) if not os.path.exists(cfg.CHECKPOINT_DIR): os.makedirs(cfg.CHECKPOINT_DIR) if not os.path.exists(cfg.SAMPLE_DIR): os.makedirs(cfg.SAMPLE_DIR) if not os.path.exists(cfg.LOGS_DIR): os.makedirs(cfg.LOGS_DIR) run_config = tf.ConfigProto() run_config.gpu_options.allow_growth = True datadir = cfg.DATASET_DIR dataset = TextDataset(datadir, 64) # dataset = TextDataset(datadir, 512) filename_test = '%s/test' % datadir dataset._test = dataset.get_data(filename_test) filename_train = '%s/train' % datadir dataset.train = dataset.get_data(filename_train) with tf.Session(config=run_config) as sess: if cfg.EVAL.FLAG: gancls = GanCls(cfg, build_model=False) eval = GanClsEval( sess=sess, model=gancls, dataset=dataset, cfg=cfg) eval.evaluate_inception() elif cfg.TRAIN.FLAG: gancls = GanCls(cfg) show_all_variables() gancls_trainer = GanClsTrainer( sess=sess, model=gancls, dataset=dataset, cfg=cfg, ) gancls_trainer.train() else: gancls = GanCls(cfg, build_model=False) gancls_visualiser = GanClsVisualizer( sess=sess, model=gancls, dataset=dataset, config=cfg, ) gancls_visualiser.visualize()
def main(_): if not FLAGS.output_file: raise ValueError( 'You must supply the path to save to with --output_file') # Read cmvn to do reverse mean variance normalization. cmvn = np.load(os.path.join(FLAGS.data_dir, "train_cmvn.npz")) with tf.Graph().as_default() as graph: model = TfModel(rnn_cell=FLAGS.rnn_cell, num_hidden=FLAGS.num_hidden, dnn_depth=FLAGS.dnn_depth, rnn_depth=FLAGS.rnn_depth, output_size=FLAGS.output_dim, bidirectional=FLAGS.bidirectional, rnn_output=FLAGS.rnn_output, cnn_output=FLAGS.cnn_output, look_ahead=FLAGS.look_ahead, mdn_output=FLAGS.mdn_output, mix_num=FLAGS.mix_num, name="tf_model") input_sequence = tf.placeholder(name='input', dtype=tf.float32, shape=[None, FLAGS.input_dim]) length = tf.expand_dims(tf.shape(input_sequence)[0], 0) # Apply normalization for input before inference. mean_inputs = tf.constant(cmvn["mean_inputs"], dtype=tf.float32) stddev_inputs = tf.constant(cmvn["stddev_inputs"], dtype=tf.float32) input_sequence = (input_sequence - mean_inputs) / stddev_inputs input_sequence = tf.expand_dims(input_sequence, 0) output_sequence_logits, final_state = model(input_sequence, length) # Apply reverse cmvn for output after inference mean_labels = tf.constant(cmvn["mean_labels"], dtype=tf.float32) stddev_labels = tf.constant(cmvn["stddev_labels"], dtype=tf.float32) output_sequence_logits = output_sequence_logits * stddev_labels + mean_labels output_sequence_logits = tf.squeeze(output_sequence_logits) output_sequence_logits = tf.identity(output_sequence_logits, name=FLAGS.output_node_name) show_all_variables() graph_def = graph.as_graph_def() with gfile.GFile(FLAGS.output_file, 'wb') as f: f.write(graph_def.SerializeToString()) #tf.train.write_graph(graph_def, './', 'inf_graph.pbtxt') tf.logging.info("Inference graph has been written to %s" % FLAGS.output_file)
def define_model(self): self.x = tf.placeholder(tf.float32, [self.cfg.TRAIN.BATCH_SIZE, 299, 299, 3], name='inputs') self.labels = tf.placeholder(tf.int32, [self.cfg.TRAIN.BATCH_SIZE]) self.logits, layers = inception_net(self.x, self.cfg.MODEL.CLASSES, for_training=True) self.pred = tf.nn.softmax(self.logits) train_correct_prediction = tf.equal( self.labels, tf.cast(tf.argmax(self.pred, 1), tf.int32)) self.train_accuracy = tf.reduce_mean( tf.cast(train_correct_prediction, tf.float32)) self.loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.labels)) self.vars_to_train = tf.trainable_variables('InceptionV3/Logits') \ + tf.trainable_variables('InceptionV3/Mixed_7c') self.all_vars = tf.global_variables('InceptionV3') self.not_to_restore = tf.global_variables('InceptionV3/Logits') \ + tf.global_variables('InceptionV3/AuxLogits') self.pretrained_to_restore = [ var for var in self.all_vars if var not in self.not_to_restore ] update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): opt = tf.train.RMSPropOptimizer(learning_rate=0.00005) self.opt_step = opt.minimize(self.loss, var_list=self.vars_to_train) self.opt_vars = [ opt.get_slot(var, name) for name in opt.get_slot_names() for var in self.vars_to_train if opt.get_slot(var, name) is not None ] show_all_variables()
def main(_): pp.pprint(flags.FLAGS.__flags) cfg = config_from_yaml(FLAGS.cfg) if not os.path.exists(cfg.CHECKPOINT_DIR): os.makedirs(cfg.CHECKPOINT_DIR) if not os.path.exists(cfg.SAMPLE_DIR): os.makedirs(cfg.SAMPLE_DIR) if not os.path.exists(cfg.LOGS_DIR): os.makedirs(cfg.LOGS_DIR) run_config = tf.ConfigProto() run_config.gpu_options.allow_growth = True datadir = cfg.DATASET_DIR dataset = TextDataset(datadir, 1) filename_test = '%s/test' % datadir dataset._test = dataset.get_data(filename_test) filename_train = '%s/train' % datadir dataset.train = dataset.get_data(filename_train) with tf.Session(config=run_config) as sess: stage_i = ConditionalGan(cfg) show_all_variables() if cfg.TRAIN.FLAG: stage_i_trainer = ConditionalGanTrainer( sess=sess, model=stage_i, dataset=dataset, cfg=cfg, ) stage_i_trainer.train() else: pass
def build_model(self): # Define the input tensor by appending the batch size dimension to the image dimension self.iter = tf.placeholder(tf.int32, shape=None) self.learning_rate = tf.placeholder(tf.float32, shape=None) self.x = tf.placeholder(tf.float32, [self.batch_size, self.output_size, self.output_size, self.channel], name='x') self.x_mismatch = tf.placeholder(tf.float32, [self.batch_size, self.output_size, self.output_size, self.channel], name='x_mismatch') self.cond = tf.placeholder(tf.float32, [self.batch_size, self.embed_dim], name='cond') self.z = tf.placeholder(tf.float32, [self.batch_size, self.z_dim], name='z') self.epsilon = tf.placeholder(tf.float32, [self.batch_size, 1, 1, 1], name='eps') self.z_sample = tf.placeholder(tf.float32, [self.sample_num] + [self.z_dim], name='z_sample') self.cond_sample = tf.placeholder(tf.float32, [self.sample_num] + [self.embed_dim], name='cond_sample') self.G, self.mean, self.log_sigma = self.generator(self.z, self.cond, stages=self.stage, t=self.trans) self.Dg_logit = self.discriminator(self.G, self.cond, reuse=False, stages=self.stage, t=self.trans) self.Dx_logit = self.discriminator(self.x, self.cond, reuse=True, stages=self.stage, t=self.trans) self.Dxmi_logit = self.discriminator(self.x_mismatch, self.cond, reuse=True, stages=self.stage, t=self.trans) self.epsilon = tf.random_uniform([self.batch_size, 1, 1, 1], 0., 1.) self.x_hat = self.epsilon * self.G + (1. - self.epsilon) * self.x self.cond_inp = self.cond + 0.0 self.Dx_hat_logit = self.discriminator(self.x_hat, self.cond_inp, reuse=True, stages=self.stage, t=self.trans) self.sampler, _, _ = self.generator(self.z_sample, self.cond_sample, reuse=True, stages=self.stage, t=self.trans) self.alpha_assign = tf.assign(self.alpha_tra, (tf.cast(tf.cast(self.iter, tf.float32) / self.steps, tf.float32))) self.d_vars = tf.trainable_variables('d_net') self.g_vars = tf.trainable_variables('g_net') show_all_variables()
def main(_): print("FLAG1") pp.pprint(flags.FLAGS.__flags) if FLAGS.input_width is None: FLAGS.input_width = FLAGS.input_height if FLAGS.output_width is None: FLAGS.output_width = FLAGS.output_height if not os.path.exists(FLAGS.checkpoint_dir): os.makedirs(FLAGS.checkpoint_dir) if not os.path.exists(FLAGS.sample_dir): os.makedirs(FLAGS.sample_dir) #gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333) run_config = tf.ConfigProto() run_config.gpu_options.allow_growth = True # extract zipfile print(FLAGS.dataset) print(os.path.join(FLAGS.data_path, "*.zip")) source_path = glob.glob(os.path.join(FLAGS.data_path, "*.zip")) print(source_path) for i, zipped_file in enumerate(source_path): print("Extracting image zip %s of %s" % (i + 1, len(source_path))) if os.path.exists(os.path.join(FLAGS.data_path, "celebA")): print("...File already exists") else: print(zipped_file) unzip_and_save(zipped_file, FLAGS.data_path) print("...Extracted!") print("Reading from %s" % os.path.join(FLAGS.data_path, "*/*.jpg")) unzipped_data_path = os.path.join( FLAGS.data_path, "*/*.jpg") #right now we support only one dataset print(unzipped_data_path) with tf.Session(config=run_config) as sess: if FLAGS.dataset == 'mnist': dcgan = DCGAN( sess, input_width=FLAGS.input_width, input_height=FLAGS.input_height, output_width=FLAGS.output_width, output_height=FLAGS.output_height, batch_size=FLAGS.batch_size, sample_num=FLAGS.batch_size, y_dim=10, data_path=FLAGS.data_path, #glob signature dataset_type=unzipped_data_path, crop=FLAGS.crop, checkpoint_dir=FLAGS.checkpoint_dir, sample_dir=FLAGS.sample_dir) else: dcgan = DCGAN(sess, input_width=FLAGS.input_width, input_height=FLAGS.input_height, output_width=FLAGS.output_width, output_height=FLAGS.output_height, batch_size=FLAGS.batch_size, sample_num=FLAGS.batch_size, data_path=unzipped_data_path, dataset_type=FLAGS.dataset, crop=FLAGS.crop, checkpoint_dir=FLAGS.checkpoint_dir, sample_dir=FLAGS.sample_dir) show_all_variables() if FLAGS.train: dcgan.train(FLAGS) else: if not dcgan.load(FLAGS.checkpoint_dir)[0]: raise Exception("[!] Train a model first, then run test mode") # to_json("./web/js/layers.js", [dcgan.h0_w, dcgan.h0_b, dcgan.g_bn0], # [dcgan.h1_w, dcgan.h1_b, dcgan.g_bn1], # [dcgan.h2_w, dcgan.h2_b, dcgan.g_bn2], # [dcgan.h3_w, dcgan.h3_b, dcgan.g_bn3], # [dcgan.h4_w, dcgan.h4_b, None]) # Below is codes for visualization OPTION = 1 visualize(sess, dcgan, FLAGS, OPTION)
def main(_): if not FLAGS.output_file: raise ValueError( 'You must supply the path to save to with --output_file') # Read cmvn to do reverse mean variance normalization. cmvn = np.load(os.path.join(FLAGS.data_dir, "train_cmvn.npz")) with tf.Graph().as_default() as graph: model = TfModel(rnn_cell=FLAGS.rnn_cell, dnn_depth=FLAGS.dnn_depth, dnn_num_hidden=FLAGS.dnn_num_hidden, rnn_depth=FLAGS.rnn_depth, rnn_num_hidden=FLAGS.rnn_num_hidden, output_size=FLAGS.output_dim, bidirectional=FLAGS.bidirectional, rnn_output=FLAGS.rnn_output, cnn_output=FLAGS.cnn_output, look_ahead=FLAGS.look_ahead, mdn_output=FLAGS.mdn_output, mix_num=FLAGS.mix_num, name="tf_model") input_sequence = tf.placeholder(name='input', dtype=tf.float32, shape=[None, FLAGS.input_dim]) length = tf.expand_dims(tf.shape(input_sequence)[0], 0) # Apply normalization for input before inference. mean_inputs = tf.constant(cmvn["mean_inputs"], dtype=tf.float32) stddev_inputs = tf.constant(cmvn["stddev_inputs"], dtype=tf.float32) input_sequence = (input_sequence - mean_inputs) / stddev_inputs input_sequence = tf.expand_dims(input_sequence, 0) output_sequence_logits, final_state = model(input_sequence, length) # Apply reverse cmvn for output after inference mean_labels = tf.constant(cmvn["mean_labels"], dtype=tf.float32) stddev_labels = tf.constant(cmvn["stddev_labels"], dtype=tf.float32) output_sequence_logits = output_sequence_logits * stddev_labels + mean_labels output_sequence_logits = tf.squeeze(output_sequence_logits) output_sequence_logits = tf.identity(output_sequence_logits, name=FLAGS.output_node_name) show_all_variables() ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_path) if ckpt: saver = tf.train.Saver() else: tf.logging.warning("Cannot find checkpoint in {}".format( args.checkpoint)) sys.exit(-1) freeze_graph.freeze_graph_with_def_protos( input_graph_def=graph.as_graph_def(), input_saver_def=saver.as_saver_def(), input_checkpoint=ckpt.model_checkpoint_path, output_node_names=FLAGS.output_node_name, restore_op_name=None, filename_tensor_name=None, output_graph=FLAGS.output_file, clear_devices=True, initializer_nodes="", variable_names_blacklist=None) tf.logging.info("Inference graph has been written to %s" % FLAGS.output_file)
def train(cv_num_batch, tr_num_batch): with tf.Graph().as_default(): with tf.device('/cpu:0'): with tf.name_scope('input'): tr_data_list = read_list(FLAGS.tr_list_file) tr_inputs, tr_labels = get_batch(tr_data_list, FLAGS.batch_size, FLAGS.input_dim, FLAGS.output_dim, FLAGS.left_context, FLAGS.right_context, FLAGS.num_threads, FLAGS.max_epoches) cv_data_list = read_list(FLAGS.cv_list_file) cv_inputs, cv_labels = get_batch(cv_data_list, FLAGS.batch_size, FLAGS.input_dim, FLAGS.output_dim, FLAGS.left_context, FLAGS.right_context, FLAGS.num_threads, FLAGS.max_epoches) devices = [] for i in xrange(FLAGS.num_gpu): device_name = ("/gpu:%d" % i) print('Using device: ', device_name) devices.append(device_name) # Prevent exhausting all the gpu memories. config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True # execute the session with tf.Session(config=config) as sess: # Create two models with tr_inputs and cv_inputs individually. with tf.name_scope('model'): print("=======================================================") print(" Build Train model ") print("=======================================================") tr_model = SEGAN(sess, FLAGS, devices, tr_inputs, tr_labels, cross_validation=False) # tr_model and val_model should share variables print("=======================================================") print(" Build Cross-Validation model ") print("=======================================================") tf.get_variable_scope().reuse_variables() cv_model = SEGAN(sess, FLAGS, devices, cv_inputs, cv_labels, cross_validation=True) show_all_variables() init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) print("Initializing variables ...") sess.run(init) if tr_model.load(tr_model.save_dir): print("[*] Load SUCCESS") else: print("[!] Load failed, maybe begin a new model.") coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) try: cv_g_loss, cv_d_loss = eval_one_epoch(sess, coord, cv_model, cv_num_batch) print(("CROSSVAL PRERUN AVG.LOSS " "%.4F(G_loss), %.4F(D_loss)") % (cv_g_loss, cv_d_loss)) sys.stdout.flush() for epoch in xrange(FLAGS.max_epoches): start = datetime.datetime.now() tr_g_loss, tr_d_loss = train_one_epoch(sess, coord, tr_model, tr_num_batch, epoch+1) cv_g_loss, cv_d_loss = eval_one_epoch(sess, coord, cv_model, cv_num_batch) end = datetime.datetime.now() print(("Epoch %02d: TRAIN AVG.LOSS " "%.5F(G_loss, lrate(%e)), %.5F(D_loss, lrate(%e)), " "CROSSVAL AVG.LOSS " "%.5F(G_loss), %.5F(D_loss), TIME USED: %.2fmin") % ( epoch+1, tr_g_loss, tr_model.g_learning_rate, tr_d_loss, tr_model.d_learning_rate, cv_g_loss, cv_d_loss, (end-start).seconds/60.0)) sys.stdout.flush() FLAGS.d_learning_rate *= FLAGS.halving_factor FLAGS.g_learning_rate *= FLAGS.halving_factor tr_model.d_learning_rate, tr_model.g_learning_rate = \ FLAGS.d_learning_rate, FLAGS.g_learning_rate tr_model.save(tr_model.save_dir, epoch+1) except Exception, e: # Report exceptions to the coordinator. coord.request_stop(e) finally:
manual_annotation_file=None) with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: model = ClassifierModel(sess, epoch=num_epochs, batch_size=_config.BATCH_SIZE, z_dim=_config.Z_DIM, dataset_name=DATASET_NAME, beta=_config.beta, num_units_in_layer=_config.num_units, train_val_data_iterator=train_val_data_iterator, log_dir=exp.config.LOG_PATH, checkpoint_dir=exp.config.TRAINED_MODELS_PATH, result_dir=exp.config.PREDICTION_RESULTS_PATH, supervise_weight=exp.config.supervise_weight, reconstruction_weight=exp.config.reconstruction_weight ) exp.model = model # show network architecture show_all_variables() exp.train(train_val_data_iterator) train_val_data_iterator.reset_counter("train") train_val_data_iterator.reset_counter("val") exp.encode_latent_vector(train_val_data_iterator, num_epochs, "train") train_val_data_iterator.reset_counter("train") train_val_data_iterator.reset_counter("val") exp.encode_latent_vector(train_val_data_iterator, num_epochs, "val")
def train(): """Run the training of the acoustic or duration model.""" dataset_train = SequenceDataset(subset="train", config_dir=FLAGS.config_dir, data_dir=FLAGS.data_dir, batch_size=FLAGS.batch_size, input_size=FLAGS.input_dim, output_size=FLAGS.output_dim, num_threads=FLAGS.num_threads, use_bucket=True, infer=False, name="dataset_train")() dataset_valid = SequenceDataset(subset="valid", config_dir=FLAGS.config_dir, data_dir=FLAGS.data_dir, batch_size=FLAGS.batch_size, input_size=FLAGS.input_dim, output_size=FLAGS.output_dim, num_threads=FLAGS.num_threads, use_bucket=True, infer=False, name="dataset_valid")() model = TfModel(rnn_cell=FLAGS.rnn_cell, dnn_depth=FLAGS.dnn_depth, dnn_num_hidden=FLAGS.dnn_num_hidden, rnn_depth=FLAGS.rnn_depth, rnn_num_hidden=FLAGS.rnn_num_hidden, output_size=FLAGS.output_dim, bidirectional=FLAGS.bidirectional, rnn_output=FLAGS.rnn_output, cnn_output=FLAGS.cnn_output, look_ahead=FLAGS.look_ahead, mdn_output=FLAGS.mdn_output, mix_num=FLAGS.mix_num, name="tf_model") # Build a reinitializable iterator for both dataset_train and dataset_valid. iterator = tf.data.Iterator.from_structure( dataset_train.batched_dataset.output_types, dataset_train.batched_dataset.output_shapes) (input_sequence, input_sequence_length, target_sequence, target_sequence_length) = iterator.get_next() training_init_op = iterator.make_initializer(dataset_train.batched_dataset) validation_init_op = iterator.make_initializer( dataset_valid.batched_dataset) # Build the model and get the loss. output_sequence_logits, train_final_state = model(input_sequence, input_sequence_length) loss = model.loss(output_sequence_logits, target_sequence, target_sequence_length) tf.summary.scalar("loss", loss) learning_rate = tf.get_variable("learning_rate", shape=[], dtype=tf.float32, initializer=tf.constant_initializer( FLAGS.learning_rate), trainable=False) reduce_learning_rate = learning_rate.assign( learning_rate * FLAGS.reduce_learning_rate_multiplier) global_step = tf.get_variable( name="global_step", shape=[], dtype=tf.int64, initializer=tf.zeros_initializer(), trainable=False, collections=[tf.GraphKeys.GLOBAL_VARIABLES, tf.GraphKeys.GLOBAL_STEP]) # Set up optimizer with global norm clipping. trainable_variables = tf.trainable_variables() optimizer = tf.train.AdamOptimizer(learning_rate) grads, _ = tf.clip_by_global_norm(tf.gradients(loss, trainable_variables), FLAGS.max_grad_norm) train_step = optimizer.apply_gradients(zip(grads, trainable_variables), global_step=global_step) show_all_variables() merged_all = tf.summary.merge_all() saver = tf.train.Saver(max_to_keep=FLAGS.max_epochs) # Train config = tf.ConfigProto() # Prevent exhausting all the gpu memories config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: # Run init sess.run(tf.global_variables_initializer()) summary_writer = tf.summary.FileWriter( os.path.join(FLAGS.save_dir, "nnet"), sess.graph) if FLAGS.resume_training: restore_from_ckpt(sess, saver) sess.run(tf.assign(learning_rate, FLAGS.learning_rate)) # add a blank line for log readability print() sys.stdout.flush() sess.run(validation_init_op) loss_prev = eval_one_epoch(sess, loss, dataset_valid.num_batches) tf.logging.info("CROSSVAL PRERUN AVG.LOSS %.4f\n" % loss_prev) for epoch in range(FLAGS.max_epochs): # Train one epoch time_start = time.time() sess.run(training_init_op) tr_loss = train_one_epoch(sess, summary_writer, merged_all, global_step, train_step, loss, dataset_train.num_batches) time_end = time.time() used_time = time_end - time_start # Validate one epoch sess.run(validation_init_op) val_loss = eval_one_epoch(sess, loss, dataset_valid.num_batches) # Determine checkpoint path FLAGS.learning_rate = sess.run(learning_rate) cptk_name = 'nnet_epoch%d_lrate%g_tr%.4f_cv%.4f' % ( epoch + 1, FLAGS.learning_rate, tr_loss, val_loss) checkpoint_path = os.path.join(FLAGS.save_dir, "nnet", cptk_name) # accept or reject new parameters if val_loss < loss_prev: saver.save(sess, checkpoint_path) # logging training loss along with validation loss tf.logging.info("EPOCH %d: TRAIN AVG.LOSS %.4f, (lrate%g) " "CROSSVAL AVG.LOSS %.4f, TIME USED %.2f, %s" % (epoch + 1, tr_loss, FLAGS.learning_rate, val_loss, used_time, "nnet accepted")) loss_prev = val_loss else: tf.logging.info("EPOCH %d: TRAIN AVG.LOSS %.4f, (lrate%g) " "CROSSVAL AVG.LOSS %.4f, TIME USED %.2f, %s" % (epoch + 1, tr_loss, FLAGS.learning_rate, val_loss, used_time, "nnet rejected")) restore_from_ckpt(sess, saver) # Reducing learning rate. sess.run(reduce_learning_rate) # add a blank line for log readability print() sys.stdout.flush()
def decode(): """Run the decoding of the acoustic or duration model.""" with tf.device('/cpu:0'): dataset_test = SequenceDataset(subset="test", config_dir=FLAGS.config_dir, data_dir=FLAGS.data_dir, batch_size=1, input_size=FLAGS.input_dim, output_size=FLAGS.output_dim, infer=True, name="dataset_test")() model = TfModel(rnn_cell=FLAGS.rnn_cell, dnn_depth=FLAGS.dnn_depth, dnn_num_hidden=FLAGS.dnn_num_hidden, rnn_depth=FLAGS.rnn_depth, rnn_num_hidden=FLAGS.rnn_num_hidden, output_size=FLAGS.output_dim, bidirectional=FLAGS.bidirectional, rnn_output=FLAGS.rnn_output, cnn_output=FLAGS.cnn_output, look_ahead=FLAGS.look_ahead, mdn_output=FLAGS.mdn_output, mix_num=FLAGS.mix_num, name="tf_model") # Build the testing model and get test output sequence. test_iterator = dataset_test.batched_dataset.make_one_shot_iterator() input_sequence, input_sequence_length = test_iterator.get_next() test_output_sequence_logits, test_final_state = model( input_sequence, input_sequence_length) show_all_variables() saver = tf.train.Saver() # Decode. with tf.Session() as sess: # Run init sess.run(tf.global_variables_initializer()) if not restore_from_ckpt(sess, saver): sys.exit(-1) # Read cmvn to do reverse mean variance normalization cmvn = np.load(os.path.join(FLAGS.data_dir, "train_cmvn.npz")) num_batches = 0 used_time_sum = frames_sum = 0.0 while True: try: time_start = time.time() logits = sess.run(test_output_sequence_logits) time_end = time.time() used_time = time_end - time_start used_time_sum += used_time frame_num = logits.shape[1] frames_sum += frame_num # Squeeze batch dimension. logits = logits.squeeze(axis=0) if FLAGS.mdn_output: out_pi = logits[:, :FLAGS.mix_num] out_mu = logits[:, FLAGS.mix_num:( FLAGS.mix_num + FLAGS.mix_num * FLAGS.output_dim)] out_sigma = logits[:, (FLAGS.mix_num + FLAGS.mix_num * FLAGS.output_dim):] max_index_pi = out_pi.argmax(axis=1) result_mu = [] for i in xrange(out_mu.shape[0]): beg_index = max_index_pi[i] * FLAGS.output_dim end_index = (max_index_pi[i] + 1) * FLAGS.output_dim result_mu.append(out_mu[i, beg_index:end_index]) logits = np.vstack(result_mu) sequence = logits * cmvn["stddev_labels"] + cmvn["mean_labels"] out_dir_name = os.path.join(FLAGS.save_dir, "test", "cmp") out_file_name = os.path.basename( dataset_test.tfrecords_lst[num_batches]).split( '.')[0] + ".cmp" out_path = os.path.join(out_dir_name, out_file_name) write_binary_file(sequence, out_path, with_dim=False) #np.savetxt(out_path, sequence, fmt="%f") tf.logging.info( "writing inferred cmp to %s (%d frames in %.4f seconds)" % (out_path, frame_num, used_time)) num_batches += 1 except tf.errors.OutOfRangeError: break tf.logging.info("Done decoding -- epoch limit reached (%d " "frames per second)" % int(frames_sum / used_time_sum))
def main(): """main""" # parse arguments args = parse_args() if args is None: exit() # Slack Bot if args.slack: config = configparser.ConfigParser() config.read('slack.config') bot = SlackBot(token=config['SLACK']['token'], channel_name=config['SLACK']['channel_name']) else: bot = None gpu_options = tf.GPUOptions(visible_device_list=args.gpu_id, allow_growth=True) session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1, gpu_options=gpu_options, allow_soft_placement=True) # open session models = [DCGAN, WGAN_GP, BEGAN, EBGAN] with tf.Session(config=session_conf) as sess: # declare instance for GAN gan = None for model in models: if args.gan_type == model.model_name: gan = model(sess, epoch=args.epoch, batch_size=args.batch_size, z_dim=args.z_dim, dataset_name=args.dataset, compute_metrics_it=args.compute_metrics_it, checkpoint_dir=args.checkpoint_dir, result_dir=args.result_dir, log_dir=args.log_dir, gpu_id=args.gpu_id, bot=bot, redo=args.redo, verbosity=args.verbosity) if gan is None: print("<!channel> ERROR!\n\n" "[!] There is no option for " + args.gan_type) raise Exception("[!] There is no option for " + args.gan_type) # build graph gan.build_model() # show network architecture show_all_variables() # launch the graph in a session gan.train() print("[*] Training finished!") # visualize learned generator gan.visualize_results(args.epoch - 1) print("[*] Testing finished!")
def train(): tr_tfrecords_lst, tr_num_batches = read_config_file( "train_8k", FLAGS.batch_size) val_tfrecords_lst, val_num_batches = read_config_file( "cv_8k", FLAGS.batch_size) with tf.Graph().as_default(): with tf.device('/cpu:0'): with tf.name_scope('input'): tr_inputs, tr_inputs_cmvn, tr_labels1, tr_labels2, tr_lengths = get_padded_batch( tr_tfrecords_lst, FLAGS.batch_size, FLAGS.input_dim, FLAGS.output_dim, num_enqueuing_threads=FLAGS.num_threads, num_epochs=FLAGS.max_epochs) val_inputs, val_inputs_cmvn, val_labels1, val_labels2, val_lengths = get_padded_batch( val_tfrecords_lst, FLAGS.batch_size, FLAGS.input_dim, FLAGS.output_dim, num_enqueuing_threads=FLAGS.num_threads, num_epochs=FLAGS.max_epochs + 1) # Create two models with train_input and val_input individually. with tf.name_scope('model'): tr_model = LSTM(FLAGS, tr_inputs_cmvn, tr_inputs, tr_labels1, tr_labels2, tr_lengths) # tr_model and val_model should share variables tf.get_variable_scope().reuse_variables() val_model = LSTM(FLAGS, val_inputs_cmvn, val_inputs, val_labels1, val_labels2, val_lengths) show_all_variables() init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) # Prevent exhausting all the gpu memories. config = tf.ConfigProto() config.gpu_options.allow_growth = True # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.allow_soft_placement = True sess = tf.Session(config=config) sess.run(init) if FLAGS.resume_training: ckpt = tf.train.get_checkpoint_state( os.path.join(FLAGS.save_dir + 'nnet')) if ckpt and ckpt.model_checkpoint_path: tf.logging.info("restore from" + ckpt.model_checkpoint_path) tr_model.saver.restore(sess, ckpt.model_checkpoint_path) else: tf.logging.fatal("checkpoint not fo2und") coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) try: # Cross validation before training. loss_prev = eval_one_epoch(sess, coord, val_model, val_num_batches) tf.logging.info("CROSSVAL PRERUN AVG.LOSS %.4F" % loss_prev) sess.run(tf.assign(tr_model.lr, FLAGS.learning_rate)) for epoch in xrange(FLAGS.max_epochs): start_time = time.time() # Training tr_loss = train_one_epoch(sess, coord, tr_model, tr_num_batches) # Validation val_loss = eval_one_epoch(sess, coord, val_model, val_num_batches) end_time = time.time() # Determine checkpoint path ckpt_name = "nnet_iter%d_lrate%e_tr%.4f_cv%.4f" % ( epoch + 1, FLAGS.learning_rate, tr_loss, val_loss) ckpt_dir = FLAGS.save_dir + '/nnet' if not os.path.exists(ckpt_dir): os.makedirs(ckpt_dir) ckpt_path = os.path.join(ckpt_dir, ckpt_name) # Accept or reject new parameters if val_loss < loss_prev: tr_model.saver.save(sess, ckpt_path) best_path = ckpt_path # Logging train loss along with validation loss tf.logging.info( "ITERATION %d: TRAIN AVG.LOSS %.4f, (lrate%e) CROSSVAL" " AVG.LOSS %.4f, %s (%s), TIME USED: %.2fs" % (epoch + 1, tr_loss, FLAGS.learning_rate, val_loss, "nnet accepted", ckpt_name, (end_time - start_time) / 1)) else: tr_model.saver.restore(sess, best_path) tf.logging.info( "ITERATION %d: TRAIN AVG.LOSS %.4f, (lrate%e) CROSSVAL" " AVG.LOSS %.4f, %s, (%s), TIME USED: %.2fs" % (epoch + 1, tr_loss, FLAGS.learning_rate, val_loss, "nnet rejected", ckpt_name, (end_time - start_time) / 1)) # Relative loss between previous and current val_loss rel_impr = (loss_prev - val_loss) / loss_prev loss_prev = val_loss # Start halving when improvement is low if rel_impr < FLAGS.start_halving_impr: FLAGS.learning_rate *= FLAGS.halving_factor sess.run(tf.assign(tr_model.lr, FLAGS.learning_rate)) # Stopping criterion if rel_impr < FLAGS.end_halving_impr: if epoch < FLAGS.min_epochs: tf.logging.info( "we were supposed to finish, but we continue as " "min_epochs : %s" % FLAGS.min_epochs) continue else: tf.logging.info( "finished, too small rel. improvement %g" % rel_impr) break except Exception, e: # Report exceptions to the coordinator. coord.request_stop(e) finally: