def train(): tr_tfrecords_lst, tr_num_batches = read_list_file("tr_tf", FLAGS.batch_size) with tf.Graph().as_default(): with tf.device('/cpu:0'): with tf.name_scope('input'): tr_mixed, tr_labels, tr_genders, tr_lengths = get_padded_batch( tr_tfrecords_lst, FLAGS.batch_size, FLAGS.input_size * 2, FLAGS.output_size * 2, num_enqueuing_threads=FLAGS.num_threads, num_epochs=FLAGS.max_epochs) tr_inputs = tf.slice(tr_mixed, [0, 0, 0], [-1, -1, FLAGS.input_size]) tr_inputs = tf.reshape(tr_inputs, [-1, FLAGS.input_size]) with tf.name_scope('model'): tr_model = LSTM(tr_inputs, tr_lengths) # tr_model = LSTM(FLAGS, tr_inputs, tr_labels,tr_lengths,tr_genders) # tr_model and val_model should share variables tf.get_variable_scope().reuse_variables show_all_variables() init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) # Prevent exhausting all the gpu memories. config = tf.ConfigProto() config.gpu_options.allow_growth = True # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.allow_soft_placement = True #sess = tf.InteractiveSession(config=config) sess = tf.Session(config=config) sess.run(init) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) try: # Cross validation before training. # lengths = sess.run(tr_lengths) # print('lengths shape: ') # print(np.shape(lengths)) # print(lengths) # tr_mix = sess.run(tr_labels) # print('tr_mix shape: ') # print(np.shape(tr_mix)) X_mean, X_std = train_one_epoch(sess, coord, tr_model, tr_num_batches) # print(np.shape(X_mean)) np.savetxt('tr_mean_good1', X_mean, fmt="%s") np.savetxt('tr_std_good1', X_std, fmt="%s") except Exception, e: # Report exceptions to the coordinator. coord.request_stop(e) finally:
def _build(self): if not self._infer: input_sequence, target_sequence, length = get_padded_batch( file_list=self._tfrecords_lst, batch_size=self._batch_size, input_size=self._input_size, output_size=self._output_size, num_enqueuing_threads=self._num_enqueuing_threads, num_epochs=self._num_epochs, infer=self._infer) return input_sequence, target_sequence, length else: input_sequence, length = get_padded_batch( file_list=self._tfrecords_lst, batch_size=self._batch_size, input_size=self._input_size, output_size=self._output_size, num_enqueuing_threads=self._num_enqueuing_threads, num_epochs=self._num_epochs, infer=self._infer) return input_sequence, length
def decode(): """Decoding the inputs using current model.""" tfrecords_lst, num_batches = read_list_file('tt_tf', FLAGS.batch_size) with tf.Graph().as_default(): with tf.device('/cpu:0'): with tf.name_scope('input'): tt_mixed, tt_labels, tt_genders, tt_lengths = get_padded_batch( tfrecords_lst, FLAGS.batch_size, FLAGS.input_size * 2, FLAGS.output_size * 2, num_enqueuing_threads=1, num_epochs=1, shuffle=False) tt_inputs = tf.slice(tt_mixed, [0, 0, 0], [-1, -1, FLAGS.input_size]) tt_angles = tf.slice(tt_mixed, [0, 0, FLAGS.input_size], [-1, -1, -1]) # Create two models with train_input and val_input individually. with tf.name_scope('model'): model = LSTM(FLAGS, tt_inputs, tt_labels, tt_lengths, tt_genders, infer=True) init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess = tf.Session() sess.run(init) ckpt = tf.train.get_checkpoint_state(FLAGS.save_dir + '/nnet') if ckpt and ckpt.model_checkpoint_path: tf.logging.info("Restore from " + ckpt.model_checkpoint_path) model.saver.restore(sess, ckpt.model_checkpoint_path) else: tf.logging.fatal("checkpoint not fou1nd.") sys.exit(-1) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) #cmvn_filename = os.path.join(FLAGS.date_dir, "/train_cmvn.npz") #if os.path.isfile(cmvn_filename): # cmvn = np.load(cmvn_filename) #else: # tf.logging.fatal("%s not exist, exit now." % cmvn_filename) # sys.exit(-1) data_dir = FLAGS.data_dir if not os.path.exists(data_dir): os.makedirs(data_dir) processed = 0 try: for batch in xrange(num_batches): if coord.should_stop(): break if FLAGS.assign == 'def': cleaned1, cleaned2, angles, lengths = sess.run( [model._cleaned1, model._cleaned2, tt_angles, tt_lengths]) else: x1, x2 = model.get_opt_output() cleaned1, cleaned2 = sess.run([x1, x2]) spec1 = cleaned1 * np.exp(angles * 1j) spec2 = cleaned2 * np.exp(angles * 1j) #sequence = activations * cmvn['stddev_labels'] + \ # cmvn['mean_labels'] for i in range(0, FLAGS.batch_size): tffilename = tfrecords_lst[i + processed] (_, name) = os.path.split(tffilename) (partname, _) = os.path.splitext(name) wav_name1 = data_dir + '/' + partname + '_1.wav' wav_name2 = data_dir + '/' + partname + '_2.wav' wav1 = istft(spec1[i, 0:lengths[i], :], size=256, shift=128) wav2 = istft(spec2[i, 0:lengths[i], :], size=256, shift=128) audiowrite(wav1, wav_name1, 8000, True, True) audiowrite(wav2, wav_name2, 8000, True, True) processed = processed + FLAGS.batch_size if batch % 50 == 0: print(batch) except Exception, e: # Report exceptions to the coordinator. coord.request_stop(e)
def train(): tr_tfrecords_lst, tr_num_batches = read_list_file("tr_tf", FLAGS.batch_size) val_tfrecords_lst, val_num_batches = read_list_file( "cv_tf", FLAGS.batch_size) with tf.Graph().as_default(): with tf.device('/cpu:0'): with tf.name_scope('input'): tr_mixed, tr_labels, tr_genders, tr_lengths = get_padded_batch( tr_tfrecords_lst, FLAGS.batch_size, FLAGS.input_size * 2, FLAGS.output_size * 2, num_enqueuing_threads=FLAGS.num_threads, num_epochs=FLAGS.max_epochs) val_mixed, val_labels, val_genders, val_lengths = get_padded_batch( val_tfrecords_lst, FLAGS.batch_size, FLAGS.input_size * 2, FLAGS.output_size * 2, num_enqueuing_threads=FLAGS.num_threads, num_epochs=FLAGS.max_epochs + 1) tr_inputs = tf.slice(tr_mixed, [0, 0, 0], [-1, -1, FLAGS.input_size]) val_inputs = tf.slice(val_mixed, [0, 0, 0], [-1, -1, FLAGS.input_size]) with tf.name_scope('model'): tr_model = LSTM(FLAGS, tr_inputs, tr_labels, tr_lengths, tr_genders) # tr_model and val_model should share variables tf.get_variable_scope().reuse_variables() val_model = LSTM(FLAGS, val_inputs, val_labels, val_lengths, val_genders) show_all_variables() init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) # Prevent exhausting all the gpu memories. config = tf.ConfigProto() config.gpu_options.allow_growth = True # config.gpu_options.per_process_gpu_memory_fraction = 0.5 config.allow_soft_placement = True #sess = tf.InteractiveSession(config=config) sess = tf.Session(config=config) sess.run(init) if FLAGS.resume_training.lower() == 'true': ckpt = tf.train.get_checkpoint_state(FLAGS.save_dir + '/nnet') if ckpt and ckpt.model_checkpoint_path: tf.logging.info("restore from" + ckpt.model_checkpoint_path) tr_model.saver.restore(sess, ckpt.model_checkpoint_path) best_path = ckpt.model_checkpoint_path else: tf.logging.fatal("checkpoint not found") coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) try: # Cross validation before training. loss_prev = eval_one_epoch(sess, coord, val_model, val_num_batches) tf.logging.info("CROSSVAL PRERUN AVG.LOSS %.4F" % loss_prev) sess.run(tf.assign(tr_model.lr, FLAGS.learning_rate)) for epoch in xrange(FLAGS.max_epochs): start_time = time.time() # Training tr_loss = train_one_epoch(sess, coord, tr_model, tr_num_batches) # Validation val_loss = eval_one_epoch(sess, coord, val_model, val_num_batches) end_time = time.time() # Determine checkpoint path ckpt_name = "nnet_iter%d_lrate%e_tr%.4f_cv%.4f" % ( epoch + 1, FLAGS.learning_rate, tr_loss, val_loss) ckpt_dir = FLAGS.save_dir + '/nnet' if not os.path.exists(ckpt_dir): os.makedirs(ckpt_dir) ckpt_path = os.path.join(ckpt_dir, ckpt_name) # Relative loss between previous and current val_loss rel_impr = (loss_prev - val_loss) / loss_prev # Accept or reject new parameters if val_loss < loss_prev: tr_model.saver.save(sess, ckpt_path) # Logging train loss along with validation loss loss_prev = val_loss best_path = ckpt_path tf.logging.info( "ITERATION %d: TRAIN AVG.LOSS %.4f, (lrate%e) CROSSVAL" " AVG.LOSS %.4f, %s (%s), TIME USED: %.2fs" % (epoch + 1, tr_loss, FLAGS.learning_rate, val_loss, "nnet accepted", ckpt_name, (end_time - start_time) / 1)) else: tr_model.saver.restore(sess, best_path) tf.logging.info( "ITERATION %d: TRAIN AVG.LOSS %.4f, (lrate%e) CROSSVAL" " AVG.LOSS %.4f, %s, (%s), TIME USED: %.2fs" % (epoch + 1, tr_loss, FLAGS.learning_rate, val_loss, "nnet rejected", ckpt_name, (end_time - start_time) / 1)) # Start halving when improvement is low if rel_impr < FLAGS.start_halving_impr: FLAGS.learning_rate *= FLAGS.halving_factor sess.run(tf.assign(tr_model.lr, FLAGS.learning_rate)) # Stopping criterion if rel_impr < FLAGS.end_halving_impr: if epoch < FLAGS.min_epochs: tf.logging.info( "we were supposed to finish, but we continue as " "min_epochs : %s" % FLAGS.min_epochs) continue else: tf.logging.info( "finished, too small rel. improvement %g" % rel_impr) break except Exception, e: # Report exceptions to the coordinator. coord.request_stop(e) finally:
def decode(): """Decoding the inputs using current model.""" tfrecords_lst, num_batches = read_config_file('test_8k', FLAGS.batch_size) with tf.Graph().as_default(): with tf.device('/cpu:0'): with tf.name_scope('input'): tt_mixed, tt_inputs, tt_labels1, tt_labels2, tt_lengths = get_padded_batch( tfrecords_lst, FLAGS.batch_size, FLAGS.input_dim, FLAGS.output_dim, num_enqueuing_threads=1, num_epochs=1, shuffle=False) # Create two models with train_input and val_input individually. with tf.name_scope('model'): model = LSTM(FLAGS, tt_inputs, tt_mixed, tt_labels1, tt_labels2, tt_lengths, infer=True) init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess = tf.Session() sess.run(init) ckpt = tf.train.get_checkpoint_state(FLAGS.save_dir + '/nnet') if ckpt and ckpt.model_checkpoint_path: tf.logging.info("Restore from " + ckpt.model_checkpoint_path) model.saver.restore(sess, ckpt.model_checkpoint_path) else: tf.logging.fatal("checkpoint not fou1nd.") sys.exit(-1) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) #cmvn_filename = os.path.join(FLAGS.date_dir, "/train_cmvn.npz") #if os.path.isfile(cmvn_filename): # cmvn = np.load(cmvn_filename) #else: # tf.logging.fatal("%s not exist, exit now." % cmvn_filename) # sys.exit(-1) data_dir = FLAGS.data_dir if not os.path.exists(data_dir): os.makedirs(data_dir) if (FLAGS.save_mask).lower() == 'true': mask_dir = data_dir + '/mask/' os.makedirs(mask_dir) try: for batch in xrange(num_batches): if coord.should_stop(): break if FLAGS.assign == 'def': cleaned1, cleaned2, m1, m2 = sess.run([ model._cleaned1, model._cleaned2, model._activations1, model._activations2 ]) else: x1, x2 = model.get_opt_output() cleaned1, cleaned2, m1, m2 = sess.run( [x1, x2, model._activations1, model._activations2]) #sequence = activations * cmvn['stddev_labels'] + \ # cmvn['mean_labels'] tffilename = tfrecords_lst[batch] (_, name) = os.path.split(tffilename) (uttid, _) = os.path.splitext(name) (partname, _) = os.path.splitext(uttid) if (FLAGS.save_mask).lower() == 'true': np.savetxt(mask_dir + partname + '_1.mask', m1[0, :, :]) np.savetxt(mask_dir + partname + '_2.mask', m2[0, :, :]) np.save(data_dir + partname + '_1.wav', cleaned1[0, :, :]) np.save(data_dir + partname + '_2.wav', cleaned2[0, :, :]) if batch % 500 == 0: print(batch) except Exception, e: # Report exceptions to the coordinator. coord.request_stop(e)