class ChainerDQNclass:
    STATE_FRAMES = 4  # number of frames to store in the state

    def __init__(self):
        self.num_of_actions = 4

        print "Initializing DQN..."

        print "Model Building"
        self.model = Chain(l1=links.Convolution2D(self.STATE_FRAMES,
                                                  32,
                                                  ksize=8,
                                                  stride=4,
                                                  nobias=False,
                                                  wscale=0.01),
                           l2=links.Convolution2D(32,
                                                  64,
                                                  ksize=4,
                                                  stride=2,
                                                  nobias=False,
                                                  wscale=0.01),
                           l3=links.Convolution2D(64,
                                                  64,
                                                  ksize=3,
                                                  stride=1,
                                                  nobias=False,
                                                  wscale=0.01),
                           l4=links.Linear(3136, 512, wscale=0.01),
                           q_value=links.Linear(512,
                                                self.num_of_actions)).to_gpu()

        self.model_target = copy.deepcopy(self.model)

        # self.optimizer = optimizers.Adam(alpha=1e-6)
        # self.optimizer.use_cleargrads()
        # self.optimizer.setup(self.model)

    def Q_func(self, state):
        h1 = funcitons.relu(self.model.l1(state))  # scale inputs in [0.0 1.0]
        h2 = funcitons.relu(self.model.l2(h1))
        h3 = funcitons.relu(self.model.l3(h2))
        h4 = funcitons.relu(self.model.l4(h3))
        Q = self.model.q_value(h4)
        return Q

    def Q_func_target(self, state):
        h1 = funcitons.relu(
            self.model_target.l1(state))  # scale inputs in [0.0 1.0]
        h2 = funcitons.relu(self.model_target.l2(h1))
        h3 = funcitons.relu(self.model_target.l3(h2))
        h4 = funcitons.relu(self.model_target.l4(h3))
        Q = self.model_target.q_value(h4)
        return Q

    def target_model_update(self):
        self.model_target = copy.deepcopy(self.model)
Пример #2
0
    plt.tick_params(labelleft="off")


# ======================================================================
# AutoEncoderの再構成画像を描画
plt.figure(figsize=(20, 20))
num = 10
cnt = 0
ans_list = []
pred_list = []
for idx in np.random.permutation(N_test)[:num]:
    with using_config('train', False):
        xxx = x_test[idx].astype(np.float32)
        h1 = F.dropout(F.relu(model_ae.l1(Variable(xxx.reshape(1, n_dim)))))
        h2 = F.dropout(F.relu(model_ae.l2(h1)))
        y = model_ae.l3(h2)
        cnt += 1
        ans_list.append(x_test[idx])
        pred_list.append(y)

cnt = 0
for i in range(int(num / 10)):
    for j in range(10):
        img_no = i * 10 + j
        pos = (2 * i) * 10 + j
        draw_digit_ae(ans_list[img_no], pos + 1, 20, 10, "ans")

    for j in range(10):
        img_no = i * 10 + j
        pos = (2 * i + 1) * 10 + j
        draw_digit_ae(pred_list[i * 10 + j].data, pos + 1, 20, 10, "reconst.")
Пример #3
0
class SDA:
    def __init__(self,
                 rng,
                 data,
                 target,
                 n_inputs=784,
                 n_hidden=[784, 784, 784, 784, 784],
                 n_outputs=1,
                 corruption_levels=[0.1, 0.1, 0.1, 0.1, 0.1],
                 gpu=-1):

        self.model = Chain(l1=L.Linear(n_inputs, n_hidden[0]),
                           l2=L.Linear(n_hidden[0], n_hidden[1]),
                           l3=L.Linear(n_hidden[1], n_hidden[2]),
                           l4=L.Linear(n_hidden[2], n_hidden[3]),
                           l5=L.Linear(n_hidden[3], n_hidden[4]),
                           l6=L.Linear(n_hidden[4], n_outputs))

        if gpu >= 0:
            self.model.to_gpu()
            self.xp = cuda.cupy
        else:
            self.xp = np

        self.rng = rng
        self.gpu = gpu
        self.data = data
        self.target = target

        self.x_train, self.x_test = data
        self.y_train, self.y_test = target

        self.n_train = len(self.y_train)
        self.n_test = len(self.y_test)

        self.corruption_levels = corruption_levels
        self.n_inputs = n_inputs
        self.n_hidden = n_hidden
        self.n_outputs = n_outputs
        self.hidden_size = len(n_hidden)

        self.dae1 = None
        self.dae2 = None
        self.dae3 = None
        self.dae4 = None
        self.dae5 = None
        self.optimizer = None
        self.setup_optimizer()

    def setup_optimizer(self):
        self.optimizer = optimizers.Adam()
        self.optimizer.setup(self.model)

    def dae_train(self, rng, n_epoch, batchsize, dae_num, data, n_inputs,
                  n_hidden, corruption_level, gpu):
        #initialize
        dae = DA(rng=rng,
                 data=data,
                 n_inputs=n_inputs,
                 n_hidden=n_hidden,
                 corruption_level=corruption_level,
                 gpu=gpu)

        #train
        print "--------DA%d training has started!--------" % dae_num
        dae.train_and_test(n_epoch=n_epoch, batchsize=batchsize)
        dae.to_cpu()
        # compute outputs for next dAE
        tmp1 = dae.compute_hidden(data[0])
        tmp2 = dae.compute_hidden(data[1])
        if gpu >= 0:
            dae.to_gpu()
        next_inputs = [tmp1, tmp2]
        return dae, next_inputs

    def pre_train(self, n_epoch=20, batchsize=40, sda_name="SDA"):
        first_inputs = self.data
        n_epoch1 = n_epoch
        batchsize1 = batchsize

        # initialize first dAE
        self.dae1, second_inputs = self.dae_train(
            self.rng,
            n_epoch=n_epoch,
            batchsize=batchsize,
            dae_num=1,
            data=first_inputs,
            n_inputs=self.n_inputs,
            n_hidden=self.n_hidden[0],
            corruption_level=self.corruption_levels[0],
            gpu=self.gpu)

        self.dae2, third_inputs = self.dae_train(
            self.rng,
            n_epoch=int(n_epoch),
            batchsize=batchsize,
            dae_num=2,
            data=second_inputs,
            n_inputs=self.n_hidden[0],
            n_hidden=self.n_hidden[1],
            corruption_level=self.corruption_levels[1],
            gpu=self.gpu)

        self.dae3, forth_inputs = self.dae_train(
            self.rng,
            n_epoch=int(n_epoch),
            batchsize=batchsize,
            dae_num=3,
            data=third_inputs,
            n_inputs=self.n_hidden[1],
            n_hidden=self.n_hidden[2],
            corruption_level=self.corruption_levels[2],
            gpu=self.gpu)

        self.dae4, fifth_inputs = self.dae_train(
            self.rng,
            n_epoch=int(n_epoch),
            batchsize=batchsize,
            dae_num=4,
            data=forth_inputs,
            n_inputs=self.n_hidden[2],
            n_hidden=self.n_hidden[3],
            corruption_level=self.corruption_levels[3],
            gpu=self.gpu)

        self.dae5, sixth_inputs = self.dae_train(
            self.rng,
            n_epoch=int(n_epoch),
            batchsize=batchsize,
            dae_num=5,
            data=fifth_inputs,
            n_inputs=self.n_hidden[3],
            n_hidden=self.n_hidden[4],
            corruption_level=self.corruption_levels[4],
            gpu=self.gpu)

        # update model parameters
        self.model.l1 = self.dae1.model.encoder
        self.model.l2 = self.dae2.model.encoder
        self.model.l3 = self.dae3.model.encoder
        self.model.l4 = self.dae4.model.encoder
        self.model.l5 = self.dae5.model.encoder

        self.setup_optimizer()

        model_file = "%s.model" % sda_name
        state_file = "%s.state" % sda_name

        serializers.save_hdf5(model_file, self.model)
        serializers.save_hdf5(state_file, self.optimizer)

    def forward(self, x_data, y_data, train=True, output=False):
        x, t = Variable(x_data), Variable(y_data)
        h1 = F.dropout(F.relu(self.model.l1(x)), train=train)
        h2 = F.dropout(F.relu(self.model.l2(h1)), train=train)
        h3 = F.dropout(F.relu(self.model.l3(h2)), train=train)
        h4 = F.dropout(F.relu(self.model.l4(h3)), train=train)
        h5 = F.dropout(F.relu(self.model.l5(h4)), train=train)
        y = F.tanh(self.model.l6(h5))
        if output:
            return y
        else:
            return F.mean_squared_error(y, t)

    def fine_tune(self, n_epoch=20, batchsize=50):
        train_accs = []
        test_accs = []

        #早期終了用配列
        self.save_accuracy = self.xp.tile([1000.0], 100)

        #ベストLOSS定義
        self.best_loss = 1000.0

        for epoch in xrange(1, n_epoch + 1):
            print 'fine tuning epoch ', epoch

            perm = self.rng.permutation(self.n_train)
            sum_loss = 0
            for i in xrange(0, self.n_train, batchsize):
                x_batch = self.xp.asarray(self.x_train[perm[i:i + batchsize]])
                y_batch = self.xp.asarray(self.y_train[perm[i:i + batchsize]])

                real_batchsize = len(x_batch)

                self.optimizer.zero_grads()
                loss = self.forward(x_batch, y_batch)

                loss.backward()
                self.optimizer.update()

                sum_loss += float(cuda.to_cpu(loss.data)) * real_batchsize

            print 'fine tuning train mean loss={}'.format(sum_loss /
                                                          self.n_train)
            train_accs.append(sum_loss / self.n_train)

            # evaluation
            sum_loss = 0
            for i in xrange(0, self.n_test, batchsize):
                x_batch = self.xp.asarray(self.x_test[i:i + batchsize])
                y_batch = self.xp.asarray(self.y_test[i:i + batchsize])

                real_batchsize = len(x_batch)

                loss = self.forward(x_batch, y_batch, train=False)

                sum_loss += float(cuda.to_cpu(loss.data)) * real_batchsize

            print 'fine tuning test mean loss={}'.format(sum_loss /
                                                         self.n_test)
            test_accs.append(sum_loss / self.n_test)

            if sum_loss < self.best_loss:
                self.best_loss = sum_loss
                self.best_epoch = epoch
                serializers.save_hdf5('mlp.model', self.model)
                print("update best loss")

            #早期終了?
            if self.xp.mean(self.save_accuracy) < sum_loss:
                print("early stopping done")
                break

            #早期終了用配列にsum_accuracyを追加
            self.save_accuracy = self.save_accuracy[1:]
            append = self.xp.array([float(sum_loss)])
            self.save_accuracy = self.xp.hstack((self.save_accuracy, append))

        print("best_epoch: %d" % (self.best_epoch))
        serializers.load_hdf5("mlp.model", self.model)

        return train_accs, test_accs
Пример #4
0
class QNet:
    # Hyper-Parameters
    gamma = 0.95  # Discount factor
    timestep_per_episode = 5000
    initial_exploration = timestep_per_episode * 1  # Initial exploratoin. original: 5x10^4
    replay_size = 32  # Replay (batch) size
    hist_size = 2  # original: 4
    data_index = 0
    data_flag = False
    loss_log = '../playground/Assets/log/'

    def __init__(self, use_gpu, enable_controller, cnn_input_dim, feature_dim,
                 agent_count, other_input_dim, model):
        self.use_gpu = use_gpu
        self.num_of_actions = len(enable_controller)
        self.enable_controller = enable_controller
        self.cnn_input_dim = cnn_input_dim
        self.feature_dim = feature_dim
        self.agent_count = agent_count
        self.other_input_dim = other_input_dim
        self.data_size = self.timestep_per_episode
        self.loss_log_file = self.loss_log + "loss.log"
        self.loss_per_episode = 0
        self.time_of_episode = 0

        print("Initializing Q-Network...")

        if model == 'None':
            self.model = Chain(
                conv1=L.Convolution2D(3 * self.hist_size, 32, 4, stride=2),
                bn1=L.BatchNormalization(32),
                conv2=L.Convolution2D(32, 32, 4, stride=2),
                bn2=L.BatchNormalization(32),
                conv3=L.Convolution2D(32, 32, 4, stride=2),
                bn3=L.BatchNormalization(32),
                #                 conv4=L.Convolution2D(64, 64, 4, stride=2),
                #                 bn4=L.BatchNormalization(64),
                l1=L.Linear(
                    self.feature_dim + self.other_input_dim * self.hist_size,
                    128),
                l2=L.Linear(128, 128),
                l3=L.Linear(128, 96),
                l4=L.Linear(96, 64),
                q_value=L.Linear(64, self.num_of_actions))
        else:
            with open(model, 'rb') as i:
                self.model = pickle.load(i)
                self.data_size = 0
        if self.use_gpu >= 0:
            self.model.to_gpu()

        self.optimizer = optimizers.RMSpropGraves()
        self.optimizer.setup(self.model)

        # History Data :  D=[s, a, r, s_dash, end_episode_flag]
        self.d = [
            np.zeros((self.agent_count, self.data_size, self.hist_size, 128,
                      128, 3),
                     dtype=np.uint8),
            np.zeros((self.agent_count, self.data_size, self.hist_size,
                      self.other_input_dim),
                     dtype=np.uint8),
            np.zeros((self.agent_count, self.data_size), dtype=np.uint8),
            np.zeros((self.agent_count, self.data_size, 1), dtype=np.float32),
            np.zeros((self.agent_count, self.data_size, 1), dtype=np.bool)
        ]

    def _reshape_for_cnn(self, state, batch_size, hist_size, x, y):

        state_ = np.zeros((batch_size, 3 * hist_size, 128, 128),
                          dtype=np.float32)
        for i in range(batch_size):
            if self.hist_size == 1:
                state_[i] = state[i][0].transpose(2, 0, 1)
            elif self.hist_size == 2:
                state_[i] = np.c_[state[i][0], state[i][1]].transpose(2, 0, 1)
            elif self.hist_size == 4:
                state_[i] = np.c_[state[i][0], state[i][1], state[i][2],
                                  state[i][3]].transpose(2, 0, 1)

        return state_

    def forward(self, state_cnn, state_other, action, reward, state_cnn_dash,
                state_other_dash, episode_end):

        num_of_batch = state_cnn.shape[0]
        s_cnn = Variable(state_cnn)
        s_oth = Variable(state_other)
        s_cnn_dash = Variable(state_cnn_dash)
        s_oth_dash = Variable(state_other_dash)

        q = self.q_func(s_cnn, s_oth)  # Get Q-value

        max_q_dash_ = self.q_func(s_cnn_dash, s_oth_dash)
        if self.use_gpu >= 0:
            tmp = list(map(np.max, max_q_dash_.data.get()))
        else:
            tmp = list(map(np.max, max_q_dash_.data))
        max_q_dash = np.asanyarray(tmp, dtype=np.float32)
        if self.use_gpu >= 0:
            target = np.array(q.data.get(), dtype=np.float32)
        else:
            target = np.array(q.data, dtype=np.float32)

        for i in range(num_of_batch):
            tmp_ = reward[i] + (1 -
                                episode_end[i]) * self.gamma * max_q_dash[i]

            action_index = self.action_to_index(action[i])
            target[i, action_index] = tmp_

        if self.use_gpu >= 0:
            loss = F.mean_squared_error(Variable(cuda.to_gpu(target)), q)
        else:
            loss = F.mean_squared_error(Variable(target), q)

        return loss, q

    def stock_experience(self, time, state_cnn, state_other, action, reward,
                         state_cnn_dash, state_other_dash, episode_end_flag):

        for i in range(self.agent_count):
            self.d[0][i][self.data_index] = state_cnn[i].copy()
            self.d[1][i][self.data_index] = state_other[i].copy()
            self.d[2][i][self.data_index] = action[i].copy()
            self.d[3][i][self.data_index] = reward[i].copy()
            self.d[4][i][self.data_index] = episode_end_flag

        self.data_index += 1
        if self.data_index >= self.data_size:
            self.data_index -= self.data_size
            self.data_flag = True

    def experience_replay(self, time):
        if self.initial_exploration < time:
            # Pick up replay_size number of samples from the Data
            replayRobotIndex = np.random.randint(0, self.agent_count,
                                                 self.replay_size)
            if not self.data_flag:  # during the first sweep of the History Data
                replay_index = np.random.randint(0, self.data_index,
                                                 self.replay_size)
            else:
                replay_index = np.random.randint(0, self.data_size,
                                                 self.replay_size)

            s_cnn_replay = np.ndarray(shape=(self.replay_size, self.hist_size,
                                             128, 128, 3),
                                      dtype=np.float32)
            s_oth_replay = np.ndarray(shape=(self.replay_size, self.hist_size,
                                             self.other_input_dim),
                                      dtype=np.float32)
            a_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.uint8)
            r_replay = np.ndarray(shape=(self.replay_size, 1),
                                  dtype=np.float32)
            s_cnn_dash_replay = np.ndarray(shape=(self.replay_size,
                                                  self.hist_size, 128, 128, 3),
                                           dtype=np.float32)
            s_oth_dash_replay = np.ndarray(shape=(self.replay_size,
                                                  self.hist_size,
                                                  self.other_input_dim),
                                           dtype=np.float32)
            episode_end_replay = np.ndarray(shape=(self.replay_size, 1),
                                            dtype=np.bool)

            for i in range(self.replay_size):
                s_cnn_replay[i] = np.asarray(
                    (self.d[0][replayRobotIndex[i]][replay_index[i]]),
                    dtype=np.float32)
                s_oth_replay[i] = np.asarray(
                    (self.d[1][replayRobotIndex[i]][replay_index[i]]),
                    dtype=np.float32)
                a_replay[i] = self.d[2][replayRobotIndex[i]][replay_index[i]]
                r_replay[i] = self.d[3][replayRobotIndex[i]][replay_index[i]]
                if (replay_index[i] + 1 >= self.data_size):
                    s_cnn_dash_replay[i] = np.array(
                        (self.d[0][replayRobotIndex[i]][replay_index[i] + 1 -
                                                        self.data_size]),
                        dtype=np.float32)
                    s_oth_dash_replay[i] = np.array(
                        (self.d[1][replayRobotIndex[i]][replay_index[i] + 1 -
                                                        self.data_size]),
                        dtype=np.float32)
                else:
                    s_cnn_dash_replay[i] = np.array(
                        (self.d[0][replayRobotIndex[i]][replay_index[i] + 1]),
                        dtype=np.float32)
                    s_oth_dash_replay[i] = np.array(
                        (self.d[1][replayRobotIndex[i]][replay_index[i] + 1]),
                        dtype=np.float32)
                episode_end_replay[i] = self.d[4][replayRobotIndex[i]][
                    replay_index[i]]

            s_cnn_replay = self._reshape_for_cnn(s_cnn_replay,
                                                 self.replay_size,
                                                 self.hist_size, 128, 128)
            s_cnn_dash_replay = self._reshape_for_cnn(s_cnn_dash_replay,
                                                      self.replay_size,
                                                      self.hist_size, 128, 128)

            s_cnn_replay /= 255.0
            s_oth_replay /= 255.0
            s_cnn_dash_replay /= 255.0
            s_oth_dash_replay /= 255.0

            if self.use_gpu >= 0:
                s_cnn_replay = cuda.to_gpu(s_cnn_replay)
                s_oth_replay = cuda.to_gpu(s_oth_replay)
                s_cnn_dash_replay = cuda.to_gpu(s_cnn_dash_replay)
                s_oth_dash_replay = cuda.to_gpu(s_oth_dash_replay)

            # Gradient-based update
            loss, _ = self.forward(s_cnn_replay, s_oth_replay, a_replay,
                                   r_replay, s_cnn_dash_replay,
                                   s_oth_dash_replay, episode_end_replay)
            send_loss = loss.data
            with open(self.loss_log_file, 'a') as the_file:
                the_file.write(str(time) + "," + str(send_loss) + "\n")
            self.loss_per_episode += loss.data
            self.time_of_episode += 1
            self.model.zerograds()
            loss.backward()
            self.optimizer.update()

    def q_func(self, state_cnn, state_other):
        if self.use_gpu >= 0:
            num_of_batch = state_cnn.data.get().shape[0]
        else:
            num_of_batch = state_cnn.data.shape[0]

        h1 = F.tanh(self.model.bn1(self.model.conv1(state_cnn)))
        h2 = F.tanh(self.model.bn2(self.model.conv2(h1)))
        h3 = F.tanh(self.model.bn3(self.model.conv3(h2)))
        #         h4 = F.tanh(self.model.bn4(self.model.conv4(h3)))
        #         h5 = F.tanh(self.model.bn5(self.model.conv5(h4)))

        h4_ = F.concat(
            (F.reshape(h3, (num_of_batch, self.feature_dim)),
             F.reshape(state_other,
                       (num_of_batch, self.other_input_dim * self.hist_size))),
            axis=1)

        h6 = F.relu(self.model.l1(h4_))
        h7 = F.relu(self.model.l2(h6))
        h8 = F.relu(self.model.l3(h7))
        h9 = F.relu(self.model.l4(h8))
        q = self.model.q_value(h9)
        return q

    def e_greedy(self, state_cnn, state_other, epsilon, reward):
        s_cnn = Variable(state_cnn)
        s_oth = Variable(state_other)
        q = self.q_func(s_cnn, s_oth)
        q = q.data
        if self.use_gpu >= 0:
            q_ = q.get()
        else:
            q_ = q

        index_action = np.zeros((self.agent_count), dtype=np.uint8)

        print(("agent"), end=' ')
        for i in range(self.agent_count):
            if np.random.rand() < epsilon:
                index_action[i] = np.random.randint(0, self.num_of_actions)
                print(("[%02d] Random(%2d)reward(%06.2f)" %
                       (i, index_action[i], reward[i])),
                      end=' ')
            else:
                index_action[i] = np.argmax(q_[i])
                print(("[%02d]!Greedy(%2d)reward(%06.2f)" %
                       (i, index_action[i], reward[i])),
                      end=' ')
            if i % 5 == 4:
                print(("\n     "), end=' ')

        del q_

        return self.index_to_action(index_action), q

    def index_to_action(self, index_of_action):
        index = np.zeros((self.agent_count), dtype=np.uint8)
        for i in range(self.agent_count):
            index[i] = self.enable_controller[index_of_action[i]]
        return index

    def action_to_index(self, action):
        return self.enable_controller.index(action)