コード例 #1
0
    def __init__(self, B, T, g_E, g_H, d_E, d_H, d_dropout, path_pos, path_neg, g_lr=1e-3, d_lr=1e-3, n_sample=16, generate_samples=10000, init_eps=0.1):
        self.B, self.T = B, T
        self.g_E, self.g_H = g_E, g_H
        self.d_E, self.d_H = d_E, d_H
        self.d_dropout = d_dropout
        self.generate_samples = generate_samples
        self.g_lr, self.d_lr = g_lr, d_lr
        self.eps = init_eps
        self.init_eps = init_eps
        self.top = os.getcwd()
        self.path_pos = path_pos
        self.path_neg = path_neg
        
        self.g_data = GeneratorPretrainingGenerator(self.path_pos, B=B, T=T, min_count=1) # next方法产生x, y_true数据; 都是同一个数据,比如[BOS, 8, 10, 6, 3, EOS]预测[8, 10, 6, 3, EOS]
        self.d_data = DiscriminatorGenerator(path_pos=self.path_pos, path_neg=self.path_neg, B=self.B, shuffle=True) # next方法产生 pos数据和neg数据

        self.V = self.g_data.V
        self.agent = Agent(sess, B, self.V, g_E, g_H, g_lr)
        self.g_beta = Agent(sess, B, self.V, g_E, g_H, g_lr)

        self.discriminator = Discriminator(self.V, d_E, d_H, d_dropout)

        self.env = Environment(self.discriminator, self.g_data, self.g_beta, n_sample=n_sample)

        self.generator_pre = GeneratorPretraining(self.V, g_E, g_H)
コード例 #2
0
ファイル: train.py プロジェクト: yenchulin/Model
    def __init__(self,
                 B,
                 T,
                 N,
                 g_E,
                 g_H,
                 d_E,
                 d_H,
                 d_dropout,
                 g_lr=1e-3,
                 d_lr=1e-3,
                 n_sample=16,
                 generate_samples=10000,
                 init_eps=0.1):
        self.B, self.T, self.N = B, T, N
        self.g_E, self.g_H = g_E, g_H
        self.d_E, self.d_H = d_E, d_H
        self.d_dropout = d_dropout
        self.generate_samples = generate_samples
        self.g_lr, self.d_lr = g_lr, d_lr
        self.eps = init_eps
        self.init_eps = init_eps
        self.top = os.getcwd()
        self.path_pos = os.path.join(self.top, 'data', 'kokoro_parsed.txt')
        self.path_pos_sentence = os.path.join(self.top, 'data',
                                              'kokoro_parsed_sentence.txt')
        self.path_neg = os.path.join(self.top, 'data', 'save',
                                     'generated_sentences.txt')
        self.vocab = Vocab(self.path_pos)
        self.g_data = GeneratorPretrainingGenerator(path=self.path_pos,
                                                    B=B,
                                                    T=T,
                                                    N=N,
                                                    vocab=self.vocab)
        if os.path.exists(self.path_neg):
            self.d_data = DiscriminatorSentenceGenerator(
                path_pos=self.path_pos_sentence,
                path_neg=self.path_neg,
                B=B,
                T=T,
                N=N,
                vocab=self.vocab)
        self.V = self.vocab.V
        self.agent = Agent(sess, B, self.N, self.V, g_E, g_H, g_lr)
        self.g_beta = Agent(sess, B, self.N, self.V, g_E, g_H, g_lr)
        self.discriminator_sentence = DiscriminatorSentence(self.V, d_dropout)
        self.env = Environment(self.discriminator_sentence.model,
                               self.g_data,
                               self.g_beta,
                               n_sample=n_sample)

        self.generator_pre = GeneratorPretraining(self.V, T, N, g_E, g_H)
        self.g_data.model_s = self.generator_pre.model_1
        self.g_data.model_w = self.generator_pre.model_2
        self.g_data.graph = tf.get_default_graph()
コード例 #3
0
    def __init__(self,
                 B,
                 T,
                 g_E,
                 g_H,
                 d_E,
                 d_H,
                 d_dropout,
                 g_lr=1e-3,
                 d_lr=1e-3,
                 n_sample=16,
                 generate_samples=10000,
                 init_eps=0.1):
        self.B, self.T = B, T
        self.g_E, self.g_H = g_E, g_H
        self.d_E, self.d_H = d_E, d_H
        self.d_dropout = d_dropout
        self.generate_samples = generate_samples
        self.g_lr, self.d_lr = g_lr, d_lr
        self.eps = init_eps
        self.init_eps = init_eps
        self.top = os.getcwd()
        self.path_pos = os.path.join(self.top, 'data', 'data\\subset.txt')
        self.path_neg = os.path.join(self.top, 'data', 'save',
                                     'generated_sentences.txt')
        self.g_data = GeneratorPretrainingGenerator(self.path_pos,
                                                    B=B,
                                                    T=T,
                                                    min_count=1)
        if os.path.exists(self.path_neg):
            self.d_data = DiscriminatorGenerator(path_pos=self.path_pos,
                                                 path_neg=self.path_neg,
                                                 B=self.B,
                                                 shuffle=True)
        self.V = self.g_data.V
        self.agent = Agent(sess, B, self.V, g_E, g_H, g_lr)
        self.g_beta = Agent(sess, B, self.V, g_E, g_H, g_lr)
        self.discriminator = Discriminator(self.V, d_E, d_H, d_dropout)
        self.env = Environment(self.discriminator,
                               self.g_data,
                               self.g_beta,
                               n_sample=n_sample)

        self.generator_pre = GeneratorPretraining(self.V, g_E, g_H)
コード例 #4
0
class Trainer(object):
    '''
    Manage training
    '''
    def __init__(self,
                 B,
                 T,
                 g_E,
                 g_H,
                 d_E,
                 d_H,
                 d_dropout,
                 path_pos,
                 path_neg,
                 g_lr=1e-3,
                 d_lr=1e-3,
                 n_sample=16,
                 generate_samples=10000,
                 init_eps=0.1):
        self.B, self.T = B, T
        self.g_E, self.g_H = g_E, g_H
        self.d_E, self.d_H = d_E, d_H
        self.d_dropout = d_dropout
        self.generate_samples = generate_samples
        self.g_lr, self.d_lr = g_lr, d_lr
        self.eps = init_eps
        self.init_eps = init_eps
        self.top = os.getcwd()
        self.path_pos = path_pos
        self.path_neg = path_neg

        self.g_data = GeneratorPretrainingGenerator(
            self.path_pos, B=B, T=T, min_count=1
        )  # next方法产生x, y_true数据; 都是同一个数据,比如[BOS, 8, 10, 6, 3, EOS]预测[8, 10, 6, 3, EOS]
        self.d_data = DiscriminatorGenerator(
            path_pos=self.path_pos,
            path_neg=self.path_neg,
            B=self.B,
            shuffle=True)  # next方法产生 pos数据和neg数据

        self.V = self.g_data.V
        self.agent = Agent(sess, B, self.V, g_E, g_H, g_lr)
        self.g_beta = Agent(sess, B, self.V, g_E, g_H, g_lr)

        self.discriminator = Discriminator(self.V, d_E, d_H, d_dropout)

        self.env = Environment(self.discriminator,
                               self.g_data,
                               self.g_beta,
                               n_sample=n_sample)

        self.generator_pre = GeneratorPretraining(self.V, g_E, g_H)

    def pre_train(self,
                  g_epochs=3,
                  d_epochs=1,
                  g_pre_path=None,
                  d_pre_path=None,
                  g_lr=1e-3,
                  d_lr=1e-3):
        self.pre_train_generator(g_epochs=g_epochs,
                                 g_pre_path=g_pre_path,
                                 lr=g_lr)

        self.pre_train_discriminator(d_epochs=d_epochs,
                                     d_pre_path=d_pre_path,
                                     lr=d_lr)

    def pre_train_generator(self, g_epochs=3, g_pre_path=None, lr=1e-3):
        if g_pre_path is None:
            self.g_pre_path = os.path.join(self.top, 'data', 'save',
                                           'generator_pre.hdf5')
        else:
            self.g_pre_path = g_pre_path

        g_adam = Adam(lr)
        self.generator_pre.compile(g_adam, 'categorical_crossentropy')
        print('Generator pre-training')
        self.generator_pre.summary()

        self.generator_pre.fit_generator(self.g_data,
                                         steps_per_epoch=None,
                                         epochs=g_epochs)
        self.generator_pre.save_weights(self.g_pre_path)
        self.reflect_pre_train()

    def pre_train_discriminator(self, d_epochs=1, d_pre_path=None, lr=1e-3):
        if d_pre_path is None:
            self.d_pre_path = os.path.join(self.top, 'data', 'save',
                                           'discriminator_pre.hdf5')
        else:
            self.d_pre_path = d_pre_path

        print('Start Generating sentences')
        self.agent.generator.generate_samples(self.T, self.g_data,
                                              self.generate_samples,
                                              self.path_neg)

        self.d_data = DiscriminatorGenerator(path_pos=self.path_pos,
                                             path_neg=self.path_neg,
                                             B=self.B,
                                             shuffle=True)

        d_adam = Adam(lr)
        self.discriminator.compile(d_adam, 'binary_crossentropy')
        self.discriminator.summary()
        print('Discriminator pre-training')

        self.discriminator.fit_generator(self.d_data,
                                         steps_per_epoch=None,
                                         epochs=d_epochs)
        self.discriminator.save(self.d_pre_path)

    def load_pre_train(self, g_pre_path, d_pre_path):
        self.generator_pre.load_weights(g_pre_path)
        self.reflect_pre_train()
        self.discriminator.load_weights(d_pre_path)

    def load_pre_train_g(self, g_pre_path):
        self.generator_pre.load_weights(g_pre_path)
        self.reflect_pre_train()

    def load_pre_train_d(self, d_pre_path):
        self.discriminator.load_weights(d_pre_path)

    def reflect_pre_train(self):
        i = 0
        for layer in self.generator_pre.layers:
            if len(layer.get_weights()) != 0:
                w = layer.get_weights()
                self.agent.generator.layers[i].set_weights(w)
                self.g_beta.generator.layers[i].set_weights(w)
                i += 1

    def train(self,
              steps=10,
              g_steps=1,
              d_steps=1,
              d_epochs=1,
              g_weights_path='data/save/generator.pkl',
              d_weights_path='data/save/discriminator.hdf5',
              verbose=True,
              head=1):
        d_adam = Adam(self.d_lr)
        self.discriminator.compile(d_adam, 'binary_crossentropy')
        self.eps = self.init_eps
        for step in range(steps):
            # Generator training
            for _ in range(g_steps):
                rewards = np.zeros([self.B, self.T])
                self.agent.reset()
                self.env.reset()
                for t in range(self.T):
                    state = self.env.get_state()

                    action = self.agent.act(state, epsilon=0.0)

                    _next_state, reward, is_episode_end, _info = self.env.step(
                        action)
                    self.agent.generator.update(state, action, reward)
                    rewards[:, t] = reward.reshape([
                        self.B,
                    ])
                    if is_episode_end:
                        if verbose:
                            print('Reward: {:.3f}, Episode end'.format(
                                np.average(rewards)))
                            self.env.render(head=head)
                        break

            # Discriminator training
            for _ in range(d_steps):
                self.agent.generator.generate_samples(self.T, self.g_data,
                                                      self.generate_samples,
                                                      self.path_neg)
                self.d_data = DiscriminatorGenerator(path_pos=self.path_pos,
                                                     path_neg=self.path_neg,
                                                     B=self.B,
                                                     shuffle=True)
                self.discriminator.fit_generator(self.d_data,
                                                 steps_per_epoch=None,
                                                 epochs=d_epochs)

            # Update env.g_beta to agent
            self.agent.save(g_weights_path)
            self.g_beta.load(g_weights_path)

            self.discriminator.save(d_weights_path)
            self.eps = max(self.eps * (1 - float(step) / steps * 4), 1e-4)

    def save(self, g_path, d_path):
        self.agent.save(g_path)
        self.discriminator.save(d_path)

    def load(self, g_path, d_path):
        self.agent.load(g_path)
        self.g_beta.load(g_path)
        self.discriminator.load_weights(d_path)

    def test(self):
        x, y = self.d_data.next()
        pred = self.discriminator.predict(x)

        for i in range(self.B):
            txt = [self.g_data.id2word[id] for id in x[i].tolist()]

            label = y[i]
            # print('{}, {:.3f}: {}'.format(label, pred[i,0], ''.join(txt)))

    def generate_txt(self, file_name, generate_samples):
        path_neg = os.path.join(self.top, 'data', 'save', file_name)

        self.agent.generator.generate_samples(self.T, self.g_data,
                                              generate_samples, path_neg)
コード例 #5
0
ファイル: train.py プロジェクト: yenchulin/Model
class Trainer(object):
    '''
    Manage training
    '''
    def __init__(self,
                 B,
                 T,
                 N,
                 g_E,
                 g_H,
                 d_E,
                 d_H,
                 d_dropout,
                 g_lr=1e-3,
                 d_lr=1e-3,
                 n_sample=16,
                 generate_samples=10000,
                 init_eps=0.1):
        self.B, self.T, self.N = B, T, N
        self.g_E, self.g_H = g_E, g_H
        self.d_E, self.d_H = d_E, d_H
        self.d_dropout = d_dropout
        self.generate_samples = generate_samples
        self.g_lr, self.d_lr = g_lr, d_lr
        self.eps = init_eps
        self.init_eps = init_eps
        self.top = os.getcwd()
        self.path_pos = os.path.join(self.top, 'data', 'kokoro_parsed.txt')
        self.path_pos_sentence = os.path.join(self.top, 'data',
                                              'kokoro_parsed_sentence.txt')
        self.path_neg = os.path.join(self.top, 'data', 'save',
                                     'generated_sentences.txt')
        self.vocab = Vocab(self.path_pos)
        self.g_data = GeneratorPretrainingGenerator(path=self.path_pos,
                                                    B=B,
                                                    T=T,
                                                    N=N,
                                                    vocab=self.vocab)
        if os.path.exists(self.path_neg):
            self.d_data = DiscriminatorSentenceGenerator(
                path_pos=self.path_pos_sentence,
                path_neg=self.path_neg,
                B=B,
                T=T,
                N=N,
                vocab=self.vocab)
        self.V = self.vocab.V
        self.agent = Agent(sess, B, self.N, self.V, g_E, g_H, g_lr)
        self.g_beta = Agent(sess, B, self.N, self.V, g_E, g_H, g_lr)
        self.discriminator_sentence = DiscriminatorSentence(self.V, d_dropout)
        self.env = Environment(self.discriminator_sentence.model,
                               self.g_data,
                               self.g_beta,
                               n_sample=n_sample)

        self.generator_pre = GeneratorPretraining(self.V, T, N, g_E, g_H)
        self.g_data.model_s = self.generator_pre.model_1
        self.g_data.model_w = self.generator_pre.model_2
        self.g_data.graph = tf.get_default_graph()

    def pre_train(self,
                  g_epochs,
                  d_epochs,
                  g_pre_path,
                  d_pre_path,
                  g_lr=1e-3,
                  d_lr=1e-3):
        self.pre_train_generator(g_epochs=g_epochs,
                                 g_pre_path=g_pre_path,
                                 lr=g_lr)
        self.pre_train_discriminator(d_epochs=d_epochs,
                                     d_pre_path=d_pre_path,
                                     lr=d_lr)

    def pre_train_generator(self, g_epochs, g_pre_path, lr):
        self.g_pre_path = g_pre_path

        g_adam = Adam(lr)
        self.generator_pre.model.compile(g_adam,
                                         'categorical_crossentropy',
                                         sample_weight_mode="temporal")
        print('Generator pre-training')
        self.generator_pre.model.summary()

        self.generator_pre.train_on_batch(self.g_data, g_epochs)
        self.generator_pre.model.save_weights(self.g_pre_path)
        self.reflect_pre_train()

    def pre_train_discriminator(self, d_epochs, d_pre_path, lr):
        self.d_pre_path = d_pre_path

        print('Start Generating sentences')
        self.agent.generator.generate_samples(
            self.T, self.g_data, self.generate_samples, self.path_neg
        )  # agent.generator weights are set after pretraining generator

        self.d_data = DiscriminatorSentenceGenerator(
            path_pos=self.path_pos_sentence,
            path_neg=self.path_neg,
            B=self.B,
            T=self.T,
            N=self.N,
            vocab=self.vocab)

        d_adam = Adam(lr)
        self.discriminator_sentence.model.compile(d_adam,
                                                  'binary_crossentropy')
        self.discriminator_sentence.model.summary()
        print('Discriminator pre-training')

        self.discriminator_sentence.train_on_batch(self.d_data, d_epochs)
        self.discriminator_sentence.model.save(self.d_pre_path)

    def load_pre_train(self, g_pre_path, d_pre_path):
        self.generator_pre.model.load_weights(g_pre_path)
        self.reflect_pre_train()
        self.discriminator_sentence.model.load_weights(d_pre_path)

    def load_pre_train_g(self, g_pre_path):
        self.generator_pre.model.load_weights(g_pre_path)
        self.reflect_pre_train()

    def load_pre_train_d(self, d_pre_path):
        self.discriminator.load_weights(d_pre_path)

    def reflect_pre_train(self):
        i = 0
        for layer in self.generator_pre.model_1.layers + self.generator_pre.model_2.layers + self.generator_pre.model_3.layers:
            if len(layer.get_weights()) != 0:
                w = layer.get_weights()
                self.agent.generator.layers[i].set_weights(w)
                self.g_beta.generator.layers[i].set_weights(w)
                i += 1

    def train(self,
              steps=10,
              g_steps=1,
              d_steps=1,
              d_epochs=1,
              g_weights_path='data/save/generator.pkl',
              d_weights_path='data/save/discriminator.hdf5',
              head=1):
        d_adam = Adam(self.d_lr)
        self.discriminator_sentence.model.compile(d_adam,
                                                  'binary_crossentropy')
        self.eps = self.init_eps

        print("Adversarial training")
        step_loss_d = []
        step_loss_g = []
        rewards = []

        for step in range(steps):
            # Generator training
            for _ in range(g_steps):
                self.agent.reset(
                )  # set agent.generator LSTM h, c state to zero vectorss
                self.env.reset_paragraph()
                for t in range(self.T):
                    self.env.reset_sentence()
                    state = self.env.get_previous_sentence(
                        t)  # previous sentence (B, N)
                    g_loss = 0
                    reward_verbose = 0
                    for n in range(self.N):
                        action = self.agent.act(
                            state, epsilon=0.0
                        )  # a word (B, 1) ex. [[20], [2239], [word id]...] or [[0], [0], [0]...] if is the end of sentence
                        reward = self.env.step(action, t, n)  # (B, 1)
                        g_loss += self.agent.generator.update(
                            state, action, reward
                        ) / self.N  # Policy gradient, update generator LSTM h, c, parameters, calulate loss for tha whole sentence
                        reward_verbose += np.mean(
                            reward.reshape(self.B)
                        ) / self.N  # mean for the batch and each word in the sentence
                    step_loss_g.append(g_loss)
                    rewards.append(reward_verbose)

            # Discriminator training
            for _ in range(d_steps):
                self.agent.generator.generate_samples(self.T, self.g_data,
                                                      self.generate_samples,
                                                      self.path_neg)
                self.d_data = DiscriminatorSentenceGenerator(
                    path_pos=self.path_pos_sentence,
                    path_neg=self.path_neg,
                    B=self.B,
                    T=self.T,
                    N=self.N,
                    vocab=self.vocab)
                d_epoch_loss = self.discriminator_sentence.train_on_batch(
                    self.d_data, d_epochs)  # shape = (d_epochs, )
                step_loss_d.append(d_epoch_loss)

            # Reflect the weight of agent to env.g_beta (DDQN)
            self.agent.save(
                g_weights_path
            )  # agent is responds for Reinforcement Learning, acting on state
            self.g_beta.load(
                g_weights_path
            )  # g_beta is responds for Rollout Policy (Monte Carol Search..)

            self.discriminator_sentence.model.save(d_weights_path)
            self.eps = max(self.eps * (1 - float(step) / steps * 4), 1e-4)

        # Plot generator loss (a loss for each sentence)
        step_loss_g = np.array(step_loss_g)  # (steps * g_steps * T, )
        xlabelName, ylabelName = "Steps", "G Loss"
        top = os.getcwd()
        images = os.path.join(top, 'data', 'save')
        figname = os.path.join(images, 'generator_loss.png')
        plotLineChart(range(1, steps * g_steps * self.T + 1), step_loss_g,
                      xlabelName, ylabelName, figname)

        # Plot discriminator loss
        step_loss_d = np.array(step_loss_d)  # (steps * d_steps, d_epochs, )
        xlabelName, ylabelName = "Steps", "D Loss"
        top = os.getcwd()
        images = os.path.join(top, 'data', 'save')
        figname = os.path.join(images, 'discriminator_loss.png')
        plotLineChart(range(1, steps * d_steps * d_epochs + 1),
                      step_loss_d.reshape(steps * d_steps * d_epochs, ),
                      xlabelName, ylabelName, figname)

        # Plot reward
        rewards = np.array(rewards)  # (steps * g_steps * T, )
        xlabelName, ylabelName = "Steps", "Rewards"
        top = os.getcwd()
        images = os.path.join(top, 'data', 'save')
        figname = os.path.join(images, 'rewards.png')
        plotLineChart(range(1, steps * g_steps * self.T + 1), rewards,
                      xlabelName, ylabelName, figname)

    def save(self, g_path, d_path):
        self.agent.save(g_path)
        self.discriminator_sentence.model.save(d_path)

    def load(self, g_path, d_path):
        self.agent.load(g_path)
        self.g_beta.load(g_path)
        self.discriminator_sentence.model.load_weights(d_path)

    def test(self):
        x, y = self.d_data.next()
        pred = self.discriminator_sentence.model.predict(x)
        for i in range(self.B):
            txt = [self.vocab.id2word[id] for id in x[i].tolist()]
            label = y[i]
            print('{}, {:.3f}: {}'.format(label, pred[i, 0], ' '.join(txt)))