Esempio n. 1
0
def train(log_dir, config):
    config.data_paths = config.data_paths

    data_dirs = [os.path.join(data_path, "data") \
            for data_path in config.data_paths]
    num_speakers = len(data_dirs)
    config.num_test = config.num_test_per_speaker * num_speakers

    if num_speakers > 1 and hparams.model_type not in ["deepvoice", "simple"]:
        raise Exception("[!] Unkown model_type for multi-speaker: {}".format(
            config.model_type))

    commit = get_git_commit() if config.git else 'None'
    checkpoint_path = os.path.join(log_dir, 'model.ckpt')

    #log(' [*] git recv-parse HEAD:\n%s' % get_git_revision_hash())
    log('=' * 50)
    #log(' [*] dit diff:\n%s' % get_git_diff())
    log('=' * 50)
    log(' [*] Checkpoint path: %s' % checkpoint_path)
    log(' [*] Loading training data from: %s' % data_dirs)
    log(' [*] Using model: %s' % config.model_dir)
    log(hparams_debug_string())

    # Set up DataFeeder:
    coord = tf.train.Coordinator()
    with tf.variable_scope('datafeeder') as scope:
        train_feeder = DataFeeder(coord,
                                  data_dirs,
                                  hparams,
                                  config,
                                  32,
                                  data_type='train',
                                  batch_size=hparams.batch_size)
        test_feeder = DataFeeder(coord,
                                 data_dirs,
                                 hparams,
                                 config,
                                 8,
                                 data_type='test',
                                 batch_size=config.num_test)

    # Set up model:
    is_randomly_initialized = config.initialize_path is None
    global_step = tf.Variable(0, name='global_step', trainable=False)

    with tf.variable_scope('model') as scope:
        model = create_model(hparams)
        model.initialize(train_feeder.inputs,
                         train_feeder.input_lengths,
                         num_speakers,
                         train_feeder.speaker_id,
                         train_feeder.mel_targets,
                         train_feeder.linear_targets,
                         train_feeder.loss_coeff,
                         is_randomly_initialized=is_randomly_initialized)

        model.add_loss()
        model.add_optimizer(global_step)
        train_stats = add_stats(model, scope_name='stats')  # legacy

    with tf.variable_scope('model', reuse=True) as scope:
        test_model = create_model(hparams)
        test_model.initialize(test_feeder.inputs,
                              test_feeder.input_lengths,
                              num_speakers,
                              test_feeder.speaker_id,
                              test_feeder.mel_targets,
                              test_feeder.linear_targets,
                              test_feeder.loss_coeff,
                              rnn_decoder_test_mode=True,
                              is_randomly_initialized=is_randomly_initialized)
        test_model.add_loss()

    test_stats = add_stats(test_model, model, scope_name='test')
    test_stats = tf.summary.merge([test_stats, train_stats])

    # Bookkeeping:
    step = 0
    time_window = ValueWindow(100)
    loss_window = ValueWindow(100)
    saver = tf.train.Saver(max_to_keep=None, keep_checkpoint_every_n_hours=2)

    sess_config = tf.ConfigProto(log_device_placement=False,
                                 allow_soft_placement=True)
    sess_config.gpu_options.allow_growth = True

    # Train!
    #with tf.Session(config=sess_config) as sess:
    with tf.Session() as sess:
        try:
            summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
            sess.run(tf.global_variables_initializer())

            if config.load_path:
                # Restore from a checkpoint if the user requested it.
                restore_path = get_most_recent_checkpoint(config.model_dir)
                saver.restore(sess, restore_path)
                log('Resuming from checkpoint: %s at commit: %s' %
                    (restore_path, commit),
                    slack=True)
            elif config.initialize_path:
                restore_path = get_most_recent_checkpoint(
                    config.initialize_path)
                saver.restore(sess, restore_path)
                log('Initialized from checkpoint: %s at commit: %s' %
                    (restore_path, commit),
                    slack=True)

                zero_step_assign = tf.assign(global_step, 0)
                sess.run(zero_step_assign)

                start_step = sess.run(global_step)
                log('=' * 50)
                log(' [*] Global step is reset to {}'. \
                        format(start_step))
                log('=' * 50)
            else:
                log('Starting new training run at commit: %s' % commit,
                    slack=True)

            start_step = sess.run(global_step)

            train_feeder.start_in_session(sess, start_step)
            test_feeder.start_in_session(sess, start_step)

            while not coord.should_stop():
                start_time = time.time()
                step, loss, opt = sess.run(
                    [global_step, model.loss_without_coeff, model.optimize],
                    feed_dict=model.get_dummy_feed_dict())

                time_window.append(time.time() - start_time)
                loss_window.append(loss)

                message = 'Step %-7d [%.03f sec/step, loss=%.05f, avg_loss=%.05f]' % (
                    step, time_window.average, loss, loss_window.average)
                log(message, slack=(step % config.checkpoint_interval == 0))

                if loss > 100 or math.isnan(loss):
                    log('Loss exploded to %.05f at step %d!' % (loss, step),
                        slack=True)
                    raise Exception('Loss Exploded')

                if step % config.summary_interval == 0:
                    log('Writing summary at step: %d' % step)

                    feed_dict = {
                        **model.get_dummy_feed_dict(),
                        **test_model.get_dummy_feed_dict()
                    }
                    summary_writer.add_summary(
                        sess.run(test_stats, feed_dict=feed_dict), step)

                if step % config.checkpoint_interval == 0:
                    log('Saving checkpoint to: %s-%d' %
                        (checkpoint_path, step))
                    saver.save(sess, checkpoint_path, global_step=step)

                if step % config.test_interval == 0:
                    log('Saving audio and alignment...')
                    num_test = config.num_test

                    fetches = [
                        model.inputs[:num_test],
                        model.linear_outputs[:num_test],
                        model.alignments[:num_test],
                        test_model.inputs[:num_test],
                        test_model.linear_outputs[:num_test],
                        test_model.alignments[:num_test],
                    ]
                    feed_dict = {
                        **model.get_dummy_feed_dict(),
                        **test_model.get_dummy_feed_dict()
                    }

                    sequences, spectrograms, alignments, \
                            test_sequences, test_spectrograms, test_alignments = \
                                    sess.run(fetches, feed_dict=feed_dict)

                    save_and_plot(sequences[:1], spectrograms[:1],
                                  alignments[:1], log_dir, step, loss, "train")
                    save_and_plot(test_sequences, test_spectrograms,
                                  test_alignments, log_dir, step, loss, "test")

        except Exception as e:
            log('Exiting due to exception: %s' % e, slack=True)
            traceback.print_exc()
            coord.request_stop(e)
Esempio n. 2
0
def train(log_dir, args):
  commit = get_git_commit() if args.git else 'None'
  checkpoint_path = os.path.join(log_dir, 'model.ckpt')
  input_path = os.path.join(args.base_dir, args.input)
  log('Checkpoint path: %s' % checkpoint_path)
  log('Loading training data from: %s' % input_path)
  log('Using model: %s' % args.model)
  log(hparams_debug_string())

  # Set up DataFeeder:
  coord = tf.train.Coordinator()
  with tf.variable_scope('datafeeder') as scope:
    feeder = DataFeeder(coord, input_path, hparams)

  # Set up model:
  global_step = tf.Variable(0, name='global_step', trainable=False)
  with tf.variable_scope('model') as scope:
    model = create_model(args.model, hparams)
    model.initialize(feeder.inputs, feeder.input_lengths, feeder.mel_targets, feeder.linear_targets)
    model.add_loss()
    model.add_optimizer(global_step)
    stats = add_stats(model)

  # Bookkeeping:
  step = 0
  time_window = ValueWindow(100)
  loss_window = ValueWindow(100)
  saver = tf.train.Saver(max_to_keep=5, keep_checkpoint_every_n_hours=2)

  # Train!
  with tf.Session() as sess:
    try:
      summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
      sess.run(tf.global_variables_initializer())

      if args.restore_step:
        # Restore from a checkpoint if the user requested it.
        restore_path = '%s-%d' % (checkpoint_path, args.restore_step)
        saver.restore(sess, restore_path)
        log('Resuming from checkpoint: %s at commit: %s' % (restore_path, commit), slack=True)
      else:
        log('Starting new training run at commit: %s' % commit, slack=True)

      feeder.start_in_session(sess)

      while not coord.should_stop():
        start_time = time.time()
        step, loss, opt = sess.run([global_step, model.loss, model.optimize])
        time_window.append(time.time() - start_time)
        loss_window.append(loss)
        message = 'Step %-7d [%.03f sec/step, loss=%.05f, avg_loss=%.05f]' % (
          step, time_window.average, loss, loss_window.average)
        log(message, slack=(step % args.checkpoint_interval == 0))

        if loss > 100 or math.isnan(loss):
          log('Loss exploded to %.05f at step %d!' % (loss, step), slack=True)
          raise Exception('Loss Exploded')

        if step % args.summary_interval == 0:
          log('Writing summary at step: %d' % step)
          summary_writer.add_summary(sess.run(stats), step)

        if step % args.checkpoint_interval == 0:
          log('Saving checkpoint to: %s-%d' % (checkpoint_path, step))
          saver.save(sess, checkpoint_path, global_step=step)
          log('Saving audio and alignment...')
          input_seq, spectrogram, alignment = sess.run([
            model.inputs[0], model.linear_outputs[0], model.alignments[0]])
          waveform = audio.inv_spectrogram(spectrogram.T)
          audio.save_wav(waveform, os.path.join(log_dir, 'step-%d-audio.wav' % step))
          plot.plot_alignment(alignment, os.path.join(log_dir, 'step-%d-align.png' % step),
            info='%s, %s, %s, step=%d, loss=%.5f' % (args.model, commit, time_string(), step, loss))
          log('Input: %s' % sequence_to_text(input_seq))

    except Exception as e:
      log('Exiting due to exception: %s' % e, slack=True)
      traceback.print_exc()
      coord.request_stop(e)
Esempio n. 3
0
def train(log_dir, args):
    commit = get_git_commit() if args.git else 'None'
    checkpoint_path = os.path.join(log_dir, 'model.ckpt')
    input_path = os.path.join(args.base_dir, args.input)
    log('Checkpoint path: %s' % checkpoint_path)
    log('Loading training data from: %s' % input_path)
    log('Using model: %s' % args.model)
    log(hparams_debug_string())

    # Set up DataFeeder:
    coord = tf.train.Coordinator()
    with tf.variable_scope('datafeeder') as scope:
        feeder = DataFeeder(coord, input_path, hparams)

    # Set up model:
    global_step = tf.Variable(0, name='global_step', trainable=False)
    with tf.variable_scope('model') as scope:
        model = SkipNet(hparams)
        model.initialize(feeder.txt_A, feeder.txt_A_lenth, feeder.txt_B, feeder.txt_B_lenth, \
           feeder.mel_targets, feeder.image_targets)
        model.add_loss()
        model.add_optimizer(global_step)
        stats = add_stats(model)

    # Bookkeeping:
    step = 0
    time_window = ValueWindow(100)
    loss_window = ValueWindow(100)
    saver = tf.train.Saver(max_to_keep=5, keep_checkpoint_every_n_hours=2)

    # Train!
    with tf.Session() as sess:
        try:
            sess.run(tf.global_variables_initializer())

            if args.restore_step:
                # Restore from a checkpoint if the user requested it.
                restore_path = '%s-%d' % (checkpoint_path, args.restore_step)
                saver.restore(sess, restore_path)
                log('Resuming from checkpoint: %s at commit: %s' %
                    (restore_path, commit),
                    slack=True)
            else:
                log('Starting new training run at commit: %s' % commit,
                    slack=True)

            feeder.start_in_session(sess)

            while not coord.should_stop():
                start_time = time.time()
                feed_dict = {
                    self.txt_targets_A: feeder.txt_A,
                    self.txt_lenth_A: feeder.txt_A_lenth,
                    self.txt_targets_B: feeder.txt_B,
                    self.txt_lenth_B: feeder.txt_B_lenth,
                    self.mel_targets: feeder.mel_targets,
                    self.image_targets: feeder.image_targets
                }

                # iter 1: dataset A : image - text pairs
                step, img_loss, txt_B_loss, d_loss, g_loss, opt1, opt2 =\
                    sess.run(feed_dict, [global_step, model.recon_img_loss, model.recon_txt_loss_A,\
                    model.domain_d_loss, model.domain_g_loss, model.optimize_recon, model.optimize_domain])

                # iter 2: dataset B: speech-text pairs
                step, speech_loss, txt_A_loss, opt1, opt2 =\
                    sess.run(feed_dict, [global_step, model.recon_speech_loss, model.recon_txt_loss_B,\
                    model.domain_d_loss, model.domain_g_loss, model.optimize_recon, model.optimize_domain])

                time_window.append(time.time() - start_time)
                loss_window.append(loss)
                message = 'Step %-7d [%.03f sec/step, loss=%.05f, avg_loss=%.05f]' % (
                    step, time_window.average, loss, loss_window.average)
                log(message, slack=(step % args.checkpoint_interval == 0))

                if loss > 100 or math.isnan(loss):
                    log('Loss exploded to %.05f at step %d!' % (loss, step),
                        slack=True)
                    raise Exception('Loss Exploded')

                if step % args.checkpoint_interval == 0:
                    log('Saving checkpoint to: %s-%d' %
                        (checkpoint_path, step))
                    saver.save(sess, checkpoint_path, global_step=step)

        except Exception as e:
            log('Exiting due to exception: %s' % e, slack=True)
            traceback.print_exc()
            coord.request_stop(e)
Esempio n. 4
0
def train(log_dir, args, trans_ckpt_dir=None):
    commit = get_git_commit() if args.git else 'None'
    checkpoint_path = os.path.join(log_dir, 'model.ckpt')
    if trans_ckpt_dir != None:
        trans_checkpoint_path = os.path.join(trans_ckpt_dir, 'model.ckpt')

    input_path = os.path.join(args.base_dir, args.input)
    log('Checkpoint path: %s' % trans_checkpoint_path)
    log('Loading training data from: %s' % input_path)
    log('Using model: %s' % args.model)
    log(hparams_debug_string())

    # Set up DataFeeder:
    coord = tf.train.Coordinator()
    with tf.variable_scope('datafeeder') as scope:
        feeder = DataFeeder(coord, input_path, hparams)

    # Set up model:
    global_step = tf.Variable(0, name='global_step', trainable=False)
    with tf.variable_scope('model') as scope:
        model = create_model(args.model, hparams)
        model.initialize(feeder.inputs, feeder.input_lengths,
                         feeder.mel_targets, feeder.linear_targets)
        model.add_loss()
        model.add_optimizer(global_step)
        stats = add_stats(model)

    # Bookkeeping:
    step = 0
    time_window = ValueWindow(100)
    loss_window = ValueWindow(100)
    saver = tf.train.Saver(max_to_keep=5, keep_checkpoint_every_n_hours=2)

    # Train!
    with tf.Session() as sess:
        try:
            summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
            sess.run(tf.global_variables_initializer())

            if args.restore_step:
                # Restore from a checkpoint if the user requested it.
                restore_path = '%s-%d' % (trans_checkpoint_path,
                                          args.restore_step)
                saver.restore(sess, restore_path)
                log('Resuming from checkpoint: %s at commit: %s' %
                    (restore_path, commit),
                    slack=True)
            else:
                log('Starting new training run at commit: %s' % commit,
                    slack=True)

            feeder.start_in_session(sess)

            while not coord.should_stop():
                start_time = time.time()
                step, loss, opt = sess.run(
                    [global_step, model.loss, model.optimize])
                time_window.append(time.time() - start_time)
                loss_window.append(loss)
                message = 'Step %-7d [%.03f sec/step, loss=%.05f, avg_loss=%.05f]' % (
                    step, time_window.average, loss, loss_window.average)
                log(message, slack=(step % args.checkpoint_interval == 0))

                if loss > 100 or math.isnan(loss):
                    log('Loss exploded to %.05f at step %d!' % (loss, step),
                        slack=True)
                    raise Exception('Loss Exploded')

                if step % args.summary_interval == 0:
                    log('Writing summary at step: %d' % step)
                    summary_writer.add_summary(sess.run(stats), step)

                if step % args.checkpoint_interval == 0:
                    log('Saving checkpoint to: %s-%d' %
                        (checkpoint_path, step))
                    saver.save(sess, checkpoint_path, global_step=step)
                    log('Saving audio and alignment...')
                    input_seq, spectrogram, alignment = sess.run([
                        model.inputs[0], model.linear_outputs[0],
                        model.alignments[0]
                    ])
                    waveform = audio.inv_spectrogram(spectrogram.T)
                    audio.save_wav(
                        waveform,
                        os.path.join(log_dir, 'step-%d-audio.wav' % step))
                    plot.plot_alignment(
                        alignment,
                        os.path.join(log_dir, 'step-%d-align.png' % step),
                        info='%s, %s, %s, step=%d, loss=%.5f' %
                        (args.model, commit, time_string(), step, loss))
                    log('Input: %s' % sequence_to_text(input_seq))

        except Exception as e:
            log('Exiting due to exception: %s' % e, slack=True)
            traceback.print_exc()
            coord.request_stop(e)
Esempio n. 5
0
def train(log_dir, pretrain_log_dir, args):
    checkpoint_path = os.path.join(log_dir, 'model.ckpt')
    input_path = os.path.join(args.base_dir, args.input)
    log('Checkpoint path: %s' % checkpoint_path)
    log('Loading training data from: %s' % input_path)
    log('Using model: %s' % args.model)
    log(hparams_debug_string())

    # Set up DataFeeder:
    ### input_path: linear, mel, frame_num, ppgs
    coord = tf.train.Coordinator()
    with tf.variable_scope('datafeeder') as scope:
        feeder = DataFeeder(coord, input_path, hparams)

    # Set up model:
    global_step = tf.Variable(0, name='global_step', trainable=False)
    with tf.variable_scope('model') as scope:
        model = create_model(args.model, hparams)
        model.initialize(input_lengths=feeder.input_lengths,
                         mel_targets=feeder.mel_targets,
                         linear_targets=feeder.linear_targets,
                         ppgs=feeder.ppgs,
                         speakers=feeder.speakers)
        model.add_loss()
        model.add_optimizer(global_step)
        stats = add_stats(model)

    # Bookkeeping:
    step = 0
    time_window = ValueWindow(100)
    loss_window = ValueWindow(100)
    acc_window = ValueWindow(100)
    saver = tf.train.Saver(max_to_keep=5, keep_checkpoint_every_n_hours=2)

    # Train!
    with tf.Session() as sess:
        try:
            summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
            sess.run(tf.global_variables_initializer())

            try:
                if pretrain_log_dir != None:
                    checkpoint_state = tf.train.get_checkpoint_state(
                        pretrain_log_dir)
                else:
                    checkpoint_state = tf.train.get_checkpoint_state(log_dir)
                if (checkpoint_state
                        and checkpoint_state.model_checkpoint_path):
                    log('Loading checkpoint {}'.format(
                        checkpoint_state.model_checkpoint_path),
                        slack=True)
                    saver.restore(sess, checkpoint_state.model_checkpoint_path)
                else:
                    log('No model to load at {}'.format(log_dir), slack=True)
                    saver.save(sess, checkpoint_path, global_step=global_step)
            except tf.errors.OutOfRangeError as e:
                log('Cannot restore checkpoint: {}'.format(e), slack=True)

            feeder.start_in_session(sess)

            while not coord.should_stop():
                start_time = time.time()

                ### how to run training
                if args.model == 'tacotron':
                    step, loss, opt = sess.run(
                        [global_step, model.loss, model.optimize])
                    time_window.append(time.time() - start_time)
                    loss_window.append(loss)
                    message = 'Step %-7d [%.03f sec/step, loss=%.05f, avg_loss=%.05f]' % (
                        step, time_window.average, loss, loss_window.average)
                    log(message, slack=(step % args.checkpoint_interval == 0))
                elif args.model == 'nnet1':
                    step, loss, opt, ppgs, logits = sess.run([
                        global_step, model.loss, model.optimize, model.ppgs,
                        model.logits
                    ])
                    ## cal acc
                    ppgs = np.argmax(ppgs, axis=-1)  # (N, 201, )
                    logits = np.argmax(logits, axis=-1)  # (N, 201, )
                    num_hits = np.sum(np.equal(ppgs, logits))
                    num_targets = np.shape(ppgs)[0] * np.shape(ppgs)[1]
                    acc = num_hits / num_targets
                    ## summerize
                    time_window.append(time.time() - start_time)
                    loss_window.append(loss)
                    acc_window.append(acc)
                    message = 'Step %-7d [%.03f sec/step, loss=%.05f, avg_loss=%.05f, acc=%.05f, avg_acc=%.05f]' % (
                        step, time_window.average, loss, loss_window.average,
                        acc, acc_window.average)
                    log(message, slack=(step % args.checkpoint_interval == 0))
                else:
                    print('input error!!')
                    assert 1 == 0

                ### save model and logs
                if loss > 100 or math.isnan(loss):
                    log('Loss exploded to %.05f at step %d!' % (loss, step),
                        slack=True)
                    raise Exception('Loss Exploded')

                if step % args.summary_interval == 0:
                    log('Writing summary at step: %d' % step)
                    summary_writer.add_summary(sess.run(stats), step)

                if step % args.checkpoint_interval == 0:
                    log('Saving checkpoint to: %s-%d' %
                        (checkpoint_path, step))
                    saver.save(sess, checkpoint_path, global_step=step)

        except Exception as e:
            log('Exiting due to exception: %s' % e, slack=True)
            traceback.print_exc()
            coord.request_stop(e)
Esempio n. 6
0
def train(log_dir, args):
    commit = get_git_commit() if args.git else 'None'
    checkpoint_path = os.path.join(log_dir, 'model.ckpt')
    input_path = os.path.join(args.base_dir, args.input)
    log('Checkpoint path: %s' % checkpoint_path)
    log('Loading training data from: %s' % input_path)
    log('Using model: %s' % args.model)
    log(hparams_debug_string())

    ps_hosts = args.ps_hosts.split(",")
    worker_hosts = args.worker_hosts.split(",")
    cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts})
    server = tf.train.Server(cluster,
                             job_name=args.job_name,
                             task_index=args.task_index)

    # Block further graph execution if current node is parameter server
    if args.job_name == "ps":
        server.join()

    with tf.device(
            tf.train.replica_device_setter(
                worker_device="/job:worker/task:%d" % args.task_index,
                cluster=cluster)):

        # Set up DataFeeder:
        coord = tf.train.Coordinator()
        with tf.variable_scope('datafeeder') as scope:
            feeder = DataFeeder(coord, input_path, hparams)

        # Set up model:
        global_step = tf.Variable(0, name='global_step', trainable=False)
        with tf.variable_scope('model') as scope:
            model = create_model(args.model, hparams)
            model.initialize(feeder.inputs, feeder.input_lengths,
                             feeder.mel_targets, feeder.linear_targets)
            model.add_loss()
            model.add_optimizer(global_step)
            stats = add_stats(model)

        # Bookkeeping:
        step = 0
        time_window = ValueWindow(100)
        loss_window = ValueWindow(100)
        saver = tf.train.Saver(max_to_keep=5,
                               keep_checkpoint_every_n_hours=2,
                               sharded=True)

        hooks = [tf.train.StopAtStepHook(last_step=1000000)]
        # Train!
        # Monitored... automatycznie wznawia z checkpointu.
        is_chief = (args.task_index == 0)
        init_op = tf.global_variables_initializer()
        sv = tf.train.Supervisor(is_chief=(args.task_index == 0),
                                 logdir="train_logs",
                                 init_op=init_op,
                                 summary_op=stats,
                                 saver=saver,
                                 save_model_secs=600)
        with sv.managed_session(server.target) as sess:
            try:

                summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
                sess.run(init_op)

                if args.restore_step and is_chief:
                    # Restore from a checkpoint if the user requested it.
                    restore_path = '%s-%d' % (checkpoint_path,
                                              args.restore_step)
                    saver.restore(sess, restore_path)
                    log('Resuming from checkpoint: %s at commit: %s' %
                        (restore_path, commit),
                        slack=True)
                else:
                    log('Starting new training run at commit: %s' % commit,
                        slack=True)

                feeder.start_in_session(sess)

                while not coord.should_stop():
                    start_time = time.time()
                    step, loss, opt = sess.run(
                        [global_step, model.loss, model.optimize])
                    time_window.append(time.time() - start_time)
                    loss_window.append(loss)
                    message = 'Step %-7d [%.03f sec/step, loss=%.05f, avg_loss=%.05f]' % (
                        step, time_window.average, loss, loss_window.average)
                    log(message, slack=(step % args.checkpoint_interval == 0))

                    if loss > 100 or math.isnan(loss):
                        log('Loss exploded to %.05f at step %d!' %
                            (loss, step),
                            slack=True)
                        raise Exception('Loss Exploded')

                    if step % args.summary_interval == 0:
                        log('Writing summary at step: %d' % step)
                        summary_writer.add_summary(sess.run(stats), step)

                    if step % args.checkpoint_interval == 0 and is_chief:
                        log('Saving checkpoint to: %s-%d' %
                            (checkpoint_path, step))
                        saver.save(sess, checkpoint_path, global_step=step)
                        log('Saving audio and alignment...')
                        input_seq, spectrogram, alignment = sess.run([
                            model.inputs[0], model.linear_outputs[0],
                            model.alignments[0]
                        ])
                        waveform = audio.inv_spectrogram(spectrogram.T)
                        audio.save_wav(
                            waveform,
                            os.path.join(log_dir, 'step-%d-audio.wav' % step))
                        plot.plot_alignment(
                            alignment,
                            os.path.join(log_dir, 'step-%d-align.png' % step),
                            info='%s, %s, %s, step=%d, loss=%.5f' %
                            (args.model, commit, time_string(), step, loss))
                        log('Input: %s' % sequence_to_text(input_seq))

            except Exception as e:
                log('Exiting due to exception: %s' % e, slack=True)
                traceback.print_exc()
                coord.request_stop(e)
Esempio n. 7
0
def train(log_dir, args):
    commit = get_git_commit() if args.git else 'None'
    checkpoint_path = os.path.join(log_dir, 'model.ckpt')
    ## input path is lists of both postive path and negtiva path
    input_path_pos = os.path.join(args.base_dir, args.input_pos)
    input_path_neg = os.path.join(args.base_dir, args.input_neg)

    log('Checkpoint path: %s' % checkpoint_path)
    log('Loading positive training data from: %s' % input_path_pos)
    log('Loading negative training data from: %s' % input_path_neg)
    log('Using model: %s' % args.model)
    log(hparams_debug_string())

    # Set up DataFeeder:
    coord = tf.train.Coordinator()
    with tf.variable_scope('datafeeder') as scope:
        feeder = DataFeeder(coord, input_path_pos, input_path_neg, hparams)

    # Set up model:
    global_step = tf.Variable(0, name='global_step', trainable=False)
    with tf.variable_scope('model') as scope:
        model = create_model(args.model, hparams)
        model.initialize(feeder.inputs_pos, feeder.input_lengths_pos,
                         feeder.mel_targets_pos, feeder.linear_targets_pos,
                         feeder.mel_targets_neg, feeder.linear_targets_neg,
                         feeder.labels_pos, feeder.labels_neg)
        model.add_loss()
        model.add_optimizer(global_step)

    # Bookkeeping:
    step = 0
    time_window = ValueWindow(100)
    loss_window = ValueWindow(100)
    saver = tf.train.Saver(max_to_keep=5, keep_checkpoint_every_n_hours=2)

    # Train!
    with tf.Session() as sess:
        try:
            #summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
            sess.run(tf.global_variables_initializer())

            if args.restore_step:
                # Restore from a checkpoint if the user requested it.
                restore_path = '%s-%d' % (checkpoint_path, args.restore_step)
                saver.restore(sess, restore_path)
                log('Resuming from checkpoint: %s at commit: %s' %
                    (restore_path, commit),
                    slack=True)
            else:
                log('Starting new training run at commit: %s' % commit,
                    slack=True)

            feeder.start_in_session(sess)

            while not coord.should_stop():
                start_time = time.time()
                # train d
                sess.run(model.d_optimize)
                # train g
                step, rec_loss, style_loss, d_loss, g_loss, _ = sess.run([
                    global_step, model.rec_loss, model.style_loss,
                    model.d_loss, model.g_loss, model.g_optimize
                ])
                time_window.append(time.time() - start_time)
                message = 'Step %-7d [%.03f sec/step, rec_loss=%.05f, style_loss=%.05f, d_loss=%.05f, g_loss=%.05f]' % (
                    step, time_window.average, rec_loss, style_loss, d_loss,
                    g_loss)
                log(message, slack=(step % args.checkpoint_interval == 0))

                if step % args.checkpoint_interval == 0:
                    log('Saving checkpoint to: %s-%d' %
                        (checkpoint_path, step))
                    saver.save(sess, checkpoint_path, global_step=step)
                    log('Saving audio and alignment...')
                    input_seq, spectrogram_pos, spectrogram_neg, alignment_pos, alignment_neg = sess.run(
                        [
                            model.inputs[0], model.linear_outputs_pos[0],
                            model.linear_outputs_neg[0],
                            model.alignments_pos[0], model.alignments_neg[0]
                        ])

                    waveform_pos = audio.inv_spectrogram(spectrogram_pos.T)
                    waveform_neg = audio.inv_spectrogram(spectrogram_neg.T)
                    audio.save_wav(
                        waveform_pos,
                        os.path.join(log_dir, 'step-%d-audio_pos.wav' % step))
                    audio.save_wav(
                        waveform_neg,
                        os.path.join(log_dir, 'step-%d-audio_neg.wav' % step))
                    plot.plot_alignment(
                        alignment_pos,
                        os.path.join(log_dir, 'step-%d-align_pos.png' % step),
                        info='%s, %s, %s, step=%d, loss=%.5f' %
                        (args.model, commit, time_string(), step, rec_loss))
                    plot.plot_alignment(
                        alignment_neg,
                        os.path.join(log_dir, 'step-%d-align_neg.png' % step),
                        info='%s, %s, %s, step=%d, loss=%.5f' %
                        (args.model, commit, time_string(), step, rec_loss))
                    log('Input: %s' % sequence_to_text(input_seq))

        except Exception as e:
            log('Exiting due to exception: %s' % e, slack=True)
            traceback.print_exc()
            coord.request_stop(e)
Esempio n. 8
0
def train(log_dir, config):
    config.data_paths = config.data_paths  # 파싱된 명령행 인자값 중 데이터 경로 : default='datasets/kr_example'

    data_dirs = [os.path.join(data_path, "data") \
            for data_path in config.data_paths]
    num_speakers = len(data_dirs) # 학습하는 화자 수 측정 : 단일화자 모델-1, 다중화자 모델-2
    config.num_test = config.num_test_per_speaker * num_speakers

    if num_speakers > 1 and hparams.model_type not in ["deepvoice", "simple"]:  # 다중화자 모델 학습일 때 모델 타입이 "deepvoice"나 "simple"이 아니라면
        raise Exception("[!] Unkown model_type for multi-speaker: {}".format(config.model_type))  # hparams.modle_type을 config.model_type으로 오타남.

    commit = get_git_commit() if config.git else 'None'  # git 관련된거여서 무시
    checkpoint_path = os.path.join(log_dir, 'model.ckpt')  # checkpoint_path 경로 지정-model.skpt 파일 경로

    log(' [*] git recv-parse HEAD:\n%s' % get_git_revision_hash())  # git log
    log('='*50)  # 줄 구분용 =====
    #log(' [*] dit diff:\n%s' % get_git_diff())
    log('='*50)  # 줄 구분용 =====
    log(' [*] Checkpoint path: %s' % checkpoint_path)  # check_point 경로 출력
    log(' [*] Loading training data from: %s' % data_dirs)
    log(' [*] Using model: %s' % config.model_dir)
    log(hparams_debug_string())

    # Set up DataFeeder:
    coord = tf.train.Coordinator()  # 쓰레드 사용 선언
    with tf.variable_scope('datafeeder') as scope:
        train_feeder = DataFeeder(
                coord, data_dirs, hparams, config, 32,
                data_type='train', batch_size=hparams.batch_size)
        # def __init__(self, coordinator, data_dirs, hparams, config, batches_per_group, data_type, batch_size):
        test_feeder = DataFeeder(
                coord, data_dirs, hparams, config, 8,
                data_type='test', batch_size=config.num_test)

    # Set up model:
    is_randomly_initialized = config.initialize_path is None
    global_step = tf.Variable(0, name='global_step', trainable=False)

    with tf.variable_scope('model') as scope:
        model = create_model(hparams)  # Tacotron 모델 생성
        model.initialize(
                train_feeder.inputs, train_feeder.input_lengths,
                num_speakers,  train_feeder.speaker_id,
                train_feeder.mel_targets, train_feeder.linear_targets,
                train_feeder.loss_coeff,
                is_randomly_initialized=is_randomly_initialized)

        model.add_loss()
        model.add_optimizer(global_step)
        train_stats = add_stats(model, scope_name='stats') # legacy

    with tf.variable_scope('model', reuse=True) as scope:
        test_model = create_model(hparams)  # Tacotron test모델 생성
        test_model.initialize(
                test_feeder.inputs, test_feeder.input_lengths,
                num_speakers, test_feeder.speaker_id,
                test_feeder.mel_targets, test_feeder.linear_targets,
                test_feeder.loss_coeff, rnn_decoder_test_mode=True,
                is_randomly_initialized=is_randomly_initialized)
        test_model.add_loss()

    test_stats = add_stats(test_model, model, scope_name='test')  # model의 loss값같은것들을 tensorboard에 기록 / model에 test_model, model2에 model
    test_stats = tf.summary.merge([test_stats, train_stats])

    # Bookkeeping:
    step = 0
    time_window = ValueWindow(100)  # ValueWindow 클래스 window_size = 100
    loss_window = ValueWindow(100)
    saver = tf.train.Saver(max_to_keep=None, keep_checkpoint_every_n_hours=2)  # 2시간에 한번씩 자동저장, checkpoint 삭제 안됨

    sess_config = tf.ConfigProto(
            log_device_placement=False,  # log_device_placement 작성하는동안 할당장치 알려줌.
            allow_soft_placement=True)  # allow_soft_placement False면 GPU없을때 오류남
    sess_config.gpu_options.allow_growth=True  # 탄력적으로 GPU메모리 사용

    # Train!
    #with tf.Session(config=sess_config) as sess:
    with tf.Session() as sess:  # with문 내의 모든 명령들은 CPU 혹은 GPU 사용 선언
        try:
            summary_writer = tf.summary.FileWriter(log_dir, sess.graph)  # summary 오퍼레이션이 평가된 결과 및 텐서보드 그래프를 파라미터 형식으로 log_dir 에 저장
            sess.run(tf.global_variables_initializer())  # 데이터셋이 로드되고 그래프가 모두 정의되면 변수를 초기화하여 훈련 시작

            if config.load_path:  # log의 설정 값들 경로를 지정하였다면
                # Restore from a checkpoint if the user requested it.
                restore_path = get_most_recent_checkpoint(config.model_dir)  # 가장 마지막에 저장된 파일경로 저장
                saver.restore(sess, restore_path)  # restore_path 값 가져오기
                log('Resuming from checkpoint: %s at commit: %s' % (restore_path, commit), slack=True)  # git과 slack을 이용한 log 출력
            elif config.initialize_path:  # log의 설정 값들로 초기화하여 사용하기로 지정하였다면
                restore_path = get_most_recent_checkpoint(config.initialize_path)  # 지정된 경로에서 가장 마지막에 저장된 파일경로 저장
                saver.restore(sess, restore_path)  # restore_path 값 가져오기
                log('Initialized from checkpoint: %s at commit: %s' % (restore_path, commit), slack=True)  # git과 slack을 이용한 log 출력

                zero_step_assign = tf.assign(global_step, 0)  # global_step의 텐서 객체 참조 변수 값을 0으로 바꿔주는 명령어 지정
                sess.run(zero_step_assign)  # 변수들을 모두 0으로 바꾸는 명령어 실행

                start_step = sess.run(global_step)  # global_step 값 부분을 시작지점으로 하여 연산 시작
                log('='*50)
                log(' [*] Global step is reset to {}'. \
                        format(start_step))  # 즉, 연산 시작 부분이 0으로 초기화 되었다고 알려줌.
                log('='*50)
            else:
                log('Starting new training run at commit: %s' % commit, slack=True)  # 과거의 데이터를 사용하지 않을 경우 새로운 학습이라고 log 출력

            start_step = sess.run(global_step)  # 연산 시작지점 가져오기

            train_feeder.start_in_session(sess, start_step)
            test_feeder.start_in_session(sess, start_step)

            while not coord.should_stop():  # 쓰레드가 멈춰야하는 상황이 아니라면
                start_time = time.time()  # 시작시간 지정(1970년 1월 1일 이후 경과된 시간을 UTC 기준으로 초로 반환)
                step, loss, opt = sess.run(
                        [global_step, model.loss_without_coeff, model.optimize],
                        feed_dict=model.get_dummy_feed_dict())  # step 값은 global_step 값으로 지정, loss 값은

                time_window.append(time.time() - start_time)
                loss_window.append(loss)

                message = 'Step %-7d [%.03f sec/step, loss=%.05f, avg_loss=%.05f]' % (
                        step, time_window.average, loss, loss_window.average)
                log(message, slack=(step % config.checkpoint_interval == 0))

                if loss > 100 or math.isnan(loss):
                    log('Loss exploded to %.05f at step %d!' % (loss, step), slack=True)
                    raise Exception('Loss Exploded')

                if step % config.summary_interval == 0:
                    log('Writing summary at step: %d' % step)

                    feed_dict = {
                            **model.get_dummy_feed_dict(),
                            **test_model.get_dummy_feed_dict()
                    }
                    summary_writer.add_summary(sess.run(
                            test_stats, feed_dict=feed_dict), step)

                if step % config.checkpoint_interval == 0:
                    log('Saving checkpoint to: %s-%d' % (checkpoint_path, step))
                    saver.save(sess, checkpoint_path, global_step=step)

                if step % config.test_interval == 0:
                    log('Saving audio and alignment...')
                    num_test = config.num_test

                    fetches = [
                            model.inputs[:num_test],
                            model.linear_outputs[:num_test],
                            model.alignments[:num_test],
                            test_model.inputs[:num_test],
                            test_model.linear_outputs[:num_test],
                            test_model.alignments[:num_test],
                    ]
                    feed_dict = {
                            **model.get_dummy_feed_dict(),
                            **test_model.get_dummy_feed_dict()
                    }

                    sequences, spectrograms, alignments, \
                            test_sequences, test_spectrograms, test_alignments = \
                                    sess.run(fetches, feed_dict=feed_dict)

                    save_and_plot(sequences[:1], spectrograms[:1], alignments[:1],
                            log_dir, step, loss, "train")
                    save_and_plot(test_sequences, test_spectrograms, test_alignments,
                            log_dir, step, loss, "test")

        except Exception as e:
            log('Exiting due to exception: %s' % e, slack=True)
            traceback.print_exc()
            coord.request_stop(e)
Esempio n. 9
0
def gst_train(log_dir, args):
	commit = get_git_commit() if args.git else 'None'
	save_dir = os.path.join(log_dir, 'gst_pretrained/')
	checkpoint_path = os.path.join(save_dir, 'gst_model.ckpt')
	input_path = os.path.join(args.base_dir, args.gst_input)
	plot_dir = os.path.join(log_dir, 'plots')
	wav_dir = os.path.join(log_dir, 'wavs')
	mel_dir = os.path.join(log_dir, 'mel-spectrograms')
	eval_dir = os.path.join(log_dir, 'eval-dir')
	eval_plot_dir = os.path.join(eval_dir, 'plots')
	eval_wav_dir = os.path.join(eval_dir, 'wavs')
	os.makedirs(eval_dir, exist_ok=True)
	os.makedirs(plot_dir, exist_ok=True)
	os.makedirs(wav_dir, exist_ok=True)
	os.makedirs(mel_dir, exist_ok=True)
	os.makedirs(eval_plot_dir, exist_ok=True)
	os.makedirs(eval_wav_dir, exist_ok=True)

	log('Checkpoint path: %s' % checkpoint_path)
	log('Loading training data from: %s' % input_path)
	log(hparams_debug_string())

	#Start by setting a seed for repeatability
	tf.set_random_seed(hparams.random_seed)
	
	# Set up DataFeeder:
	coord = tf.train.Coordinator()
	with tf.variable_scope('datafeeder') as scope:
		feeder = DataFeeder(coord, input_path, hparams)

	# Set up model:
	global_step = tf.Variable(0, name='global_step', trainable=False)
	model, stats = model_train_mode(args, feeder, hparams, global_step)
	eval_model = model_test_mode(args, feeder, hparams, global_step)
		
	# Bookkeeping:
	step = 0
	time_window = ValueWindow(100)
	loss_window = ValueWindow(100)
	saver = tf.train.Saver(max_to_keep=5, keep_checkpoint_every_n_hours=2)

	#Memory allocation on the GPU as needed
	config = tf.ConfigProto()
	config.gpu_options.allow_growth = True


	# Train!
	with tf.Session(config=config) as sess:
		try:
			summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
			sess.run(tf.global_variables_initializer())
			checkpoint_state = False
			#saved model restoring
			if args.restore_step:
				#Restore saved model if the user requested it, Default = True.
				try:
					checkpoint_state = tf.train.get_checkpoint_state(save_dir)
				except tf.errors.OutOfRangeError as e:
					log('Cannot restore checkpoint: {}'.format(e))

			if (checkpoint_state and checkpoint_state.model_checkpoint_path):
				log('Loading checkpoint {}'.format(checkpoint_state.model_checkpoint_path))
				saver.restore(sess, checkpoint_state.model_checkpoint_path)

			else:
				if not args.restore_step:
					log('Starting new training!')
				else:
					log('No model to load at {}'.format(save_dir))

			feeder.start_in_session(sess)

			while not coord.should_stop() and step < args.gst_train_steps:
				start_time = time.time()
				step, loss, opt = sess.run([global_step, model.loss, model.optimize])
				time_window.append(time.time() - start_time)
				loss_window.append(loss)
				message = 'Step %-7d [%.03f sec/step, loss=%.05f, avg_loss=%.05f]' % (
					step, time_window.average, loss, loss_window.average)
				log(message, slack=(step % args.checkpoint_interval == 0))

				if loss > 100 or math.isnan(loss):
					log('Loss exploded to %.05f at step %d!' % (loss, step), slack=True)
					raise Exception('Loss Exploded')

				if step % args.summary_interval == 0:
					log('Writing summary at step: %d' % step)
					summary_writer.add_summary(sess.run(stats), step)

				if step % args.eval_interval == 0:
					#Run eval and save eval stats
					log('\nRunning evaluation at step {}'.format(step))

					eval_losses = []
					linear_losses = []

					#TODO: FIX TO ENCOMPASS MORE LOSS
					for i in tqdm(range(feeder.test_steps)):
									eloss, linear_loss, mel_p, mel_t, t_len, align, lin_p = sess.run([eval_model.loss, eval_model.linear_loss, 
										eval_model.mel_outputs[0], eval_model.mel_targets[0], eval_model.targets_lengths[0], eval_model.alignments[0], 
										eval_model.linear_outputs[0]])
									eval_losses.append(eloss)
									linear_losses.append(linear_loss)


					eval_loss = sum(eval_losses) / len(eval_losses)
					linear_loss = sum(linear_losses) / len(linear_losses)

					wav = audio.inv_linear_spectrogram(lin_p.T)
					audio.save_wav(wav, os.path.join(eval_wav_dir, 'step-{}-eval-waveform-linear.wav'.format(step)))
					log('Saving eval log to {}..'.format(eval_dir))
					#Save some log to monitor model improvement on same unseen sequence

					wav = audio.inv_mel_spectrogram(mel_p.T)
					audio.save_wav(wav, os.path.join(eval_wav_dir, 'step-{}-eval-waveform-mel.wav'.format(step)))

					plot.plot_alignment(align, os.path.join(eval_plot_dir, 'step-{}-eval-align.png'.format(step)),
									info='{}, {}, step={}, loss={:.5f}'.format(args.model, time_string(), step, eval_loss),
									max_len=t_len // hparams.outputs_per_step)
					plot.plot_spectrogram(mel_p, os.path.join(eval_plot_dir, 'step-{}-eval-mel-spectrogram.png'.format(step)),
									info='{}, {}, step={}, loss={:.5}'.format(args.model, time_string(), step, eval_loss), target_spectrogram=mel_t,
									)

					log('Eval loss for global step {}: {:.3f}'.format(step, eval_loss))
					log('Writing eval summary!')
					add_eval_stats(summary_writer, step, linear_loss, eval_loss)      

				if step % args.checkpoint_interval == 0 or step == args.gst_train_steps:
					log('Saving checkpoint to: %s-%d' % (checkpoint_path, step))
					saver.save(sess, checkpoint_path, global_step=step)
					log('Saving audio and alignment...')
					input_seq, mel_pred, alignment, target, target_len = sess.run([model.inputs[0],
						model.mel_outputs[0],
						model.alignments[0],
						model.mel_targets[0],
						model.targets_lengths[0],
						])
					
					#save predicted mel spectrogram to disk (debug)
					mel_filename = 'mel-prediction-step-{}.npy'.format(step)
					np.save(os.path.join(mel_dir, mel_filename), mel_pred.T, allow_pickle=False)

					#save griffin lim inverted wav for debug (mel -> wav)
					wav = audio.inv_mel_spectrogram(mel_pred.T)
					audio.save_wav(wav, os.path.join(wav_dir, 'step-{}-wave-from-mel.wav'.format(step)))

					#save alignment plot to disk (control purposes)
					plot.plot_alignment(alignment, os.path.join(plot_dir, 'step-{}-align.png'.format(step)),
									info='{}, {}, step={}, loss={:.5f}'.format(args.model, time_string(), step, loss),
									max_len=target_len // hparams.outputs_per_step)
					#save real and predicted mel-spectrogram plot to disk (control purposes)
					plot.plot_spectrogram(mel_pred, os.path.join(plot_dir, 'step-{}-mel-spectrogram.png'.format(step)),
									info='{}, {}, step={}, loss={:.5}'.format(args.model, time_string(), step, loss), target_spectrogram=target,
									max_len=target_len)
					log('Input at step {}: {}'.format(step, sequence_to_text(input_seq)))

			log('GST Taco training complete after {} global steps!'.format(args.gst_train_steps))
			return save_dir

		
		except Exception as e:
			log('Exiting due to exception: %s' % e, slack=True)
			traceback.print_exc()
			coord.request_stop(e)
Esempio n. 10
0
def train(log_dir, args):
    commit = get_git_commit() if args.git else 'None'
    checkpoint_path = os.path.join(log_dir, 'model.ckpt')
    input_path = os.path.join(args.base_dir, args.input)
    log('Checkpoint path: %s' % checkpoint_path)
    log('Loading training data from: %s' % input_path)
    log('Using model: %s' % args.model)
    log(hparams_debug_string())

    # Set up DataFeeder:
    coord = tf.train.Coordinator()
    with tf.variable_scope('datafeeder') as _:
        feeder = DataFeeder(coord, input_path, hparams)

    # Set up model:
    global_step = tf.Variable(0, name='global_step', trainable=False)
    with tf.variable_scope('model') as _:
        model = create_model(args.model, hparams)
        model.initialize(feeder.inputs, args.vgg19_pretrained_model,
                         feeder.mel_targets, feeder.linear_targets)
        model.add_loss()
        model.add_optimizer(global_step)
        stats = add_stats(model)

    # Bookkeeping:
    time_window = ValueWindow()
    loss_window = ValueWindow()
    saver = tf.train.Saver(keep_checkpoint_every_n_hours=2)

    # Train!
    with tf.Session() as sess:
        try:
            train_start_time = time.time()
            summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
            sess.run(tf.global_variables_initializer())

            if args.restore_step:
                # Restore from a checkpoint if the user requested it.
                restore_path = '%s-%d' % (checkpoint_path, args.restore_step)
                checkpoint_saver = tf.train.import_meta_graph(
                    '%s.%s' % (restore_path, 'meta'))
                checkpoint_saver.restore(sess, restore_path)
                log('Resuming from checkpoint: %s at commit: %s' %
                    (restore_path, commit))
            else:
                log('Starting new training run at commit: %s' % commit)

            feeder.start_in_session(sess)

            while not coord.should_stop():
                start_time = time.time()
                step, loss, opt = sess.run(
                    [global_step, model.loss, model.optimize])
                time_window.append(time.time() - start_time)
                loss_window.append(loss)
                message = 'Step %-7d [%.03f sec/step, loss=%.05f, avg_loss=%.05f]' % (
                    step, time_window.average, loss, loss_window.average)
                log(message, slack=(step % args.summary_interval == 0))

                if loss > 100 or math.isnan(loss):
                    log('Loss exploded to %.05f at step %d!' % (loss, step))
                    raise Exception('Loss Exploded')

                if step % args.summary_interval == 0:
                    log('Writing summary at step: %d' % step)
                    summary_writer.add_summary(sess.run(stats), step)

                if step % args.checkpoint_interval == 0:
                    log('Saving checkpoint to: %s-%d' %
                        (checkpoint_path, step))
                    saver.save(sess, checkpoint_path, global_step=step)
                    log('Saving audio...')
                    _, spectrogram, _ = sess.run([
                        model.inputs[0], model.linear_outputs[0],
                        model.alignments[0]
                    ])
                    waveform = audio.inv_spectrogram(spectrogram.T)
                    audio_path = os.path.join(log_dir,
                                              'step-%d-audio.wav' % step)
                    audio.save_wav(waveform, audio_path)

                    infolog.upload_to_slack(audio_path, step)

                    time_so_far = time.time() - train_start_time
                    hrs, rest = divmod(time_so_far, 3600)
                    min, secs = divmod(rest, 60)
                    log('{:.0f} hrs, {:.0f}mins and {:.1f}sec since the training process began'
                        .format(hrs, min, secs))

                if asked_to_stop(step):
                    coord.request_stop()

        except Exception as e:
            log('@channel: Exiting due to exception: %s' % e)
            traceback.print_exc()
            coord.request_stop(e)
Esempio n. 11
0
def train(log_dir, args):
    checkpoint_path = os.path.join(log_dir, 'model.ckpt')
    input_path = os.path.join(args.base_dir, args.input)
    # 显示模型的路径信息
    log('Checkpoint path: %s' % checkpoint_path)
    log('Loading training data from: %s' % input_path)
    coord = tf.train.Coordinator()
    with tf.variable_scope('datafeeder') as scope:
        feeder = DataFeeder(coord, input_path, hparams)

    # 初始化模型
    global_step = tf.Variable(0, name='global_step', trainable=False)
    with tf.variable_scope('model') as scope:
        model = Tacotron(hparams)
        model.initialize(feeder.inputs, feeder.input_lengths,
                         feeder.mel_targets, feeder.linear_targets,
                         feeder.stop_token_targets, global_step)
        model.add_loss()
        model.add_optimizer(global_step)
        stats = add_stats(model)

    step = 0
    time_window = ValueWindow(100)
    loss_window = ValueWindow(100)
    saver = tf.train.Saver(max_to_keep=1)

    # 开始训练
    with tf.Session() as sess:
        try:
            summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
            sess.run(tf.global_variables_initializer())
            feeder.start_in_session(sess)

            while not coord.should_stop():
                start_time = time.time()
                step, loss, opt = sess.run(
                    [global_step, model.loss, model.optimize])
                time_window.append(time.time() - start_time)
                loss_window.append(loss)
                message = 'Step %-7d [%.03f sec/step, loss=%.05f, avg_loss=%.05f]' % (
                    step, time_window.average, loss, loss_window.average)
                log(message, slack=(step % args.checkpoint_interval == 0))

                if step % args.summary_interval == 0:
                    summary_writer.add_summary(sess.run(stats), step)

                # 每隔一定的训练步数生成检查点
                if step % args.checkpoint_interval == 0:
                    log('Saving checkpoint to: %s-%d' %
                        (checkpoint_path, step))
                    saver.save(sess, checkpoint_path, global_step=step)
                    log('Saving audio and alignment...')
                    input_seq, spectrogram, alignment = sess.run([
                        model.inputs[0], model.linear_outputs[0],
                        model.alignments[0]
                    ])
                    waveform = audio.inv_spectrogram(spectrogram.T)
                    # 合成样音
                    audio.save_wav(
                        waveform,
                        os.path.join(log_dir, 'step-%d-audio.wav' % step))
                    time_string = datetime.now().strftime('%Y-%m-%d %H:%M')
                    # 画Encoder-Decoder对齐图
                    infolog.plot_alignment(
                        alignment,
                        os.path.join(log_dir, 'step-%d-align.png' % step),
                        info='%s,  %s, step=%d, loss=%.5f' %
                        (args.model, time_string, step, loss))
                    # 显示合成样音的文本
                    log('Input: %s' % sequence_to_text(input_seq))

        except Exception as e:
            log('Exiting due to exception: %s' % e, slack=True)
            traceback.print_exc()
            coord.request_stop(e)
Esempio n. 12
0
def train(log_dir, args):
    commit = get_git_commit() if args.git else 'None'
    checkpoint_path = os.path.join(log_dir, 'model.ckpt')
    input_path = os.path.join(args.base_dir, args.input)
    log('Checkpoint path: %s' % checkpoint_path)
    log('Loading training data from: %s' % input_path)
    log('Using model: %s' % args.model)
    log(hparams_debug_string())

    # graph
    with tf.Graph().as_default(), tf.device('/cpu:0'):

        #new attributes of hparams
        #hparams.num_GPU = len(GPUs_id)
        #hparams.datasets = eval(args.datasets)
        hparams.datasets = eval(args.datasets)
        hparams.prenet_layer1 = args.prenet_layer1
        hparams.prenet_layer2 = args.prenet_layer2
        hparams.gru_size = args.gru_size
        hparams.attention_size = args.attention_size
        hparams.rnn_size = args.rnn_size
        hparams.enable_fv1 = args.enable_fv1
        hparams.enable_fv2 = args.enable_fv2

        if args.batch_size:
            hparams.batch_size = args.batch_size

        # Multi-GPU settings
        GPUs_id = eval(args.GPUs_id)
        hparams.num_GPU = len(GPUs_id)
        tower_grads = []
        tower_loss = []
        models = []

        global_step = tf.Variable(-1, name='global_step', trainable=False)
        if hparams.decay_learning_rate:
            learning_rate = _learning_rate_decay(hparams.initial_learning_rate,
                                                 global_step, hparams.num_GPU)
        else:
            learning_rate = tf.convert_to_tensor(hparams.initial_learning_rate)
        # Set up DataFeeder:
        coord = tf.train.Coordinator()
        with tf.variable_scope('datafeeder') as scope:
            input_path = os.path.join(args.base_dir, args.input)
            feeder = DataFeeder(coord, input_path, hparams)
            inputs = feeder.inputs
            inputs = tf.split(inputs, hparams.num_GPU, 0)
            input_lengths = feeder.input_lengths
            input_lengths = tf.split(input_lengths, hparams.num_GPU, 0)
            mel_targets = feeder.mel_targets
            mel_targets = tf.split(mel_targets, hparams.num_GPU, 0)
            linear_targets = feeder.linear_targets
            linear_targets = tf.split(linear_targets, hparams.num_GPU, 0)

        # Set up model:
        with tf.variable_scope('model') as scope:
            optimizer = tf.train.AdamOptimizer(learning_rate,
                                               hparams.adam_beta1,
                                               hparams.adam_beta2)
            for i, GPU_id in enumerate(GPUs_id):
                with tf.device('/gpu:%d' % GPU_id):
                    with tf.name_scope('GPU_%d' % GPU_id):

                        if hparams.enable_fv1 or hparams.enable_fv2:
                            net = ResCNN(data=mel_targets[i],
                                         batch_size=hparams.batch_size,
                                         hyparam=hparams)
                            net.inference()

                            voice_print_feature = tf.reduce_mean(
                                net.features, 0)
                        else:
                            voice_print_feature = None

                        models.append(None)
                        models[i] = create_model(args.model, hparams)
                        models[i].initialize(
                            inputs=inputs[i],
                            input_lengths=input_lengths[i],
                            mel_targets=mel_targets[i],
                            linear_targets=linear_targets[i],
                            voice_print_feature=voice_print_feature)
                        models[i].add_loss()
                        """L2 weight decay loss."""
                        if args.weight_decay > 0:
                            costs = []
                            for var in tf.trainable_variables():
                                #if var.op.name.find(r'DW') > 0:
                                costs.append(tf.nn.l2_loss(var))
                                # tf.summary.histogram(var.op.name, var)
                            weight_decay = tf.cast(args.weight_decay,
                                                   tf.float32)
                            cost = models[i].loss
                            models[i].loss += tf.multiply(
                                weight_decay, tf.add_n(costs))
                            cost_pure_wd = tf.multiply(weight_decay,
                                                       tf.add_n(costs))
                        else:
                            cost = models[i].loss
                            cost_pure_wd = tf.constant([0])

                        tower_loss.append(models[i].loss)

                        tf.get_variable_scope().reuse_variables()
                        models[i].add_optimizer(global_step, optimizer)

                        tower_grads.append(models[i].gradients)

            # calculate average gradient
            gradients = average_gradients(tower_grads)

            stats = add_stats(models[0], gradients, learning_rate)
            time.sleep(10)

        # apply average gradient

        with tf.control_dependencies(tf.get_collection(
                tf.GraphKeys.UPDATE_OPS)):
            apply_gradient_op = optimizer.apply_gradients(
                gradients, global_step=global_step)

        # Bookkeeping:
        step = 0
        time_window = ValueWindow(100)
        loss_window = ValueWindow(100)
        saver = tf.train.Saver(max_to_keep=5, keep_checkpoint_every_n_hours=2)

        # Train!
        config = tf.ConfigProto(log_device_placement=False,
                                allow_soft_placement=True)
        config.gpu_options.allow_growth = True
        with tf.Session(config=config) as sess:
            try:
                summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
                sess.run(tf.global_variables_initializer())

                if args.restore_step:
                    # Restore from a checkpoint if the user requested it.
                    restore_path = '%s-%d' % (checkpoint_path,
                                              args.restore_step)
                    saver.restore(sess, restore_path)
                    log('Resuming from checkpoint: %s at commit: %s' %
                        (restore_path, commit),
                        slack=True)
                else:
                    log('Starting new training run at commit: %s' % commit,
                        slack=True)

                feeder.start_in_session(sess)

                while not coord.should_stop():
                    start_time = time.time()
                    model = models[0]

                    step, loss, opt, loss_wd, loss_pure_wd = sess.run([
                        global_step, cost, apply_gradient_op, model.loss,
                        cost_pure_wd
                    ])
                    feeder._batch_in_queue -= 1
                    log('feed._batch_in_queue: %s' %
                        str(feeder._batch_in_queue),
                        slack=True)

                    time_window.append(time.time() - start_time)
                    loss_window.append(loss)
                    message = 'Step %-7d [%.03f sec/step, loss=%.05f, avg_loss=%.05f, loss_wd=%.05f, loss_pure_wd=%.05f]' % (
                        step, time_window.average, loss, loss_window.average,
                        loss_wd, loss_pure_wd)
                    log(message, slack=(step % args.checkpoint_interval == 0))

                    #if the gradient seems to explode, then restore to the previous step
                    if loss > 2 * loss_window.average or math.isnan(loss):
                        log('recover to the previous checkpoint')
                        #tf.reset_default_graph()
                        restore_step = int(
                            (step - 10) / args.checkpoint_interval
                        ) * args.checkpoint_interval
                        restore_path = '%s-%d' % (checkpoint_path,
                                                  restore_step)
                        saver.restore(sess, restore_path)
                        continue

                    if loss > 100 or math.isnan(loss):
                        log('Loss exploded to %.05f at step %d!' %
                            (loss, step),
                            slack=True)
                        raise Exception('Loss Exploded')

                    try:
                        if step % args.summary_interval == 0:
                            log('Writing summary at step: %d' % step)
                            summary_writer.add_summary(sess.run(stats), step)
                    except:
                        pass

                    if step % args.checkpoint_interval == 0:
                        log('Saving checkpoint to: %s-%d' %
                            (checkpoint_path, step))
                        saver.save(sess, checkpoint_path, global_step=step)
                        log('Saving audio and alignment...')
                        input_seq, spectrogram, alignment = sess.run([
                            model.inputs[0], model.linear_outputs[0],
                            model.alignments[0]
                        ])
                        waveform = audio.inv_spectrogram(spectrogram.T)
                        audio.save_wav(
                            waveform,
                            os.path.join(log_dir, 'step-%d-audio.wav' % step))
                        plot.plot_alignment(
                            alignment,
                            os.path.join(log_dir, 'step-%d-align.png' % step),
                            info='%s, %s, %s, step=%d, loss=%.5f' %
                            (args.model, commit, time_string(), step, loss))
                        log('Input: %s' % sequence_to_text(input_seq))

            except Exception as e:
                log('Exiting due to exception: %s' % e, slack=True)
                traceback.print_exc()
                coord.request_stop(e)
Esempio n. 13
0
def train(log_dir, args):
    commit = get_git_commit() if args.git else 'None'
    checkpoint_path = os.path.join(log_dir, 'model.ckpt')
    DATA_PATH = {'bznsyp': "BZNSYP", 'ljspeech': "LJSpeech-1.1"}[args.dataset]
    input_path = os.path.join(args.base_dir, 'DATA', DATA_PATH, 'training',
                              'train.txt')
    log('Checkpoint path: %s' % checkpoint_path)
    log('Loading training data from: %s' % input_path)
    log('Using model: %s' % args.model)
    log(hparams_debug_string())

    # Set up DataFeeder:
    coord = tf.train.Coordinator()
    with tf.variable_scope('datafeeder') as scope:
        feeder = DataFeeder(coord, input_path, hparams)

    # Set up model:
    global_step = tf.Variable(0, name='global_step', trainable=False)
    with tf.variable_scope('model') as scope:
        model = create_model(args.model, hparams)
        model.initialize(feeder.inputs, feeder.input_lengths,
                         feeder.lpc_targets, feeder.stop_token_targets)
        model.add_loss()
        model.add_optimizer(global_step)
        stats = add_stats(model)

    # Bookkeeping:
    step = 0
    time_window = ValueWindow(100)
    loss_window = ValueWindow(100)
    saver = tf.train.Saver(max_to_keep=999, keep_checkpoint_every_n_hours=2)

    # Train!
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    with tf.Session(config=config) as sess:
        try:
            summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
            sess.run(tf.global_variables_initializer())

            if args.restore_step:
                # Restore from a checkpoint if the user requested it.
                checkpoint_state = tf.train.get_checkpoint_state(log_dir)
                # restore_path = '%s-%d' % (checkpoint_path, args.restore_step)
                if checkpoint_state is not None:
                    saver.restore(sess, checkpoint_state.model_checkpoint_path)
                    log('Resuming from checkpoint: %s at commit: %s' %
                        (checkpoint_state.model_checkpoint_path, commit),
                        slack=True)
            else:
                log('Starting new training run at commit: %s' % commit,
                    slack=True)

            if args.restore_decoder:
                models = [
                    f for f in os.listdir('pretrain') if f.find('.meta') != -1
                ]
                decoder_ckpt_path = os.path.join(
                    'pretrain', models[0].replace('.meta', ''))

                global_vars = tf.global_variables()
                var_list = []
                valid_scope = [
                    'model/inference/decoder', 'model/inference/post_cbhg',
                    'model/inference/dense', 'model/inference/memory_layer'
                ]
                for v in global_vars:
                    if v.name.find('attention') != -1:
                        continue
                    if v.name.find('Attention') != -1:
                        continue
                    for scope in valid_scope:
                        if v.name.startswith(scope):
                            var_list.append(v)
                decoder_saver = tf.train.Saver(var_list)
                decoder_saver.restore(sess, decoder_ckpt_path)
                print('restore pretrained decoder ...')

            feeder.start_in_session(sess)

            while not coord.should_stop():
                start_time = time.time()
                step, loss, opt = sess.run(
                    [global_step, model.loss, model.optimize])
                time_window.append(time.time() - start_time)
                loss_window.append(loss)
                message = 'Step %-7d [%.03f sec/step, loss=%.05f, avg_loss=%.05f]' % (
                    step, time_window.average, loss, loss_window.average)
                log(message, slack=(step % args.checkpoint_interval == 0))

                if loss > 100 or math.isnan(loss):
                    log('Loss exploded to %.05f at step %d!' % (loss, step),
                        slack=True)
                    raise Exception('Loss Exploded')

                if step % args.summary_interval == 0:
                    log('Writing summary at step: %d' % step)
                    summary_writer.add_summary(sess.run(stats), step)

                if step % args.checkpoint_interval == 0:
                    log('Saving checkpoint to: %s-%d' %
                        (checkpoint_path, step))
                    saver.save(sess, checkpoint_path, global_step=step)
                    log('Saving audio and alignment...')
                    input_seq, lpc_targets, alignment = sess.run([
                        model.inputs[0], model.lpc_outputs[0],
                        model.alignments[0]
                    ])
                    plot.plot_alignment(
                        alignment,
                        os.path.join(log_dir, 'step-%d-align.png' % step),
                        info='%s, %s, %s, step=%d, loss=%.5f' %
                        (args.model, commit, time_string(), step, loss))
                    np.save(os.path.join(log_dir, 'step-%d-lpc.npy' % step),
                            lpc_targets)
                    log('Input: %s' % sequence_to_text(input_seq))

        except Exception as e:
            log('Exiting due to exception: %s' % e, slack=True)
            traceback.print_exc()
            coord.request_stop(e)