class ChainerDQNclass: STATE_FRAMES = 4 # number of frames to store in the state def __init__(self): self.num_of_actions = 4 print "Initializing DQN..." print "Model Building" self.model = Chain(l1=links.Convolution2D(self.STATE_FRAMES, 32, ksize=8, stride=4, nobias=False, wscale=0.01), l2=links.Convolution2D(32, 64, ksize=4, stride=2, nobias=False, wscale=0.01), l3=links.Convolution2D(64, 64, ksize=3, stride=1, nobias=False, wscale=0.01), l4=links.Linear(3136, 512, wscale=0.01), q_value=links.Linear(512, self.num_of_actions)).to_gpu() self.model_target = copy.deepcopy(self.model) # self.optimizer = optimizers.Adam(alpha=1e-6) # self.optimizer.use_cleargrads() # self.optimizer.setup(self.model) def Q_func(self, state): h1 = funcitons.relu(self.model.l1(state)) # scale inputs in [0.0 1.0] h2 = funcitons.relu(self.model.l2(h1)) h3 = funcitons.relu(self.model.l3(h2)) h4 = funcitons.relu(self.model.l4(h3)) Q = self.model.q_value(h4) return Q def Q_func_target(self, state): h1 = funcitons.relu( self.model_target.l1(state)) # scale inputs in [0.0 1.0] h2 = funcitons.relu(self.model_target.l2(h1)) h3 = funcitons.relu(self.model_target.l3(h2)) h4 = funcitons.relu(self.model_target.l4(h3)) Q = self.model_target.q_value(h4) return Q def target_model_update(self): self.model_target = copy.deepcopy(self.model)
plt.tick_params(labelbottom="off") plt.tick_params(labelleft="off") # ====================================================================== # AutoEncoderの再構成画像を描画 plt.figure(figsize=(20, 20)) num = 10 cnt = 0 ans_list = [] pred_list = [] for idx in np.random.permutation(N_test)[:num]: with using_config('train', False): xxx = x_test[idx].astype(np.float32) h1 = F.dropout(F.relu(model_ae.l1(Variable(xxx.reshape(1, n_dim))))) h2 = F.dropout(F.relu(model_ae.l2(h1))) y = model_ae.l3(h2) cnt += 1 ans_list.append(x_test[idx]) pred_list.append(y) cnt = 0 for i in range(int(num / 10)): for j in range(10): img_no = i * 10 + j pos = (2 * i) * 10 + j draw_digit_ae(ans_list[img_no], pos + 1, 20, 10, "ans") for j in range(10): img_no = i * 10 + j pos = (2 * i + 1) * 10 + j
class SDA: def __init__(self, rng, data, target, n_inputs=784, n_hidden=[784, 784, 784, 784, 784], n_outputs=1, corruption_levels=[0.1, 0.1, 0.1, 0.1, 0.1], gpu=-1): self.model = Chain(l1=L.Linear(n_inputs, n_hidden[0]), l2=L.Linear(n_hidden[0], n_hidden[1]), l3=L.Linear(n_hidden[1], n_hidden[2]), l4=L.Linear(n_hidden[2], n_hidden[3]), l5=L.Linear(n_hidden[3], n_hidden[4]), l6=L.Linear(n_hidden[4], n_outputs)) if gpu >= 0: self.model.to_gpu() self.xp = cuda.cupy else: self.xp = np self.rng = rng self.gpu = gpu self.data = data self.target = target self.x_train, self.x_test = data self.y_train, self.y_test = target self.n_train = len(self.y_train) self.n_test = len(self.y_test) self.corruption_levels = corruption_levels self.n_inputs = n_inputs self.n_hidden = n_hidden self.n_outputs = n_outputs self.hidden_size = len(n_hidden) self.dae1 = None self.dae2 = None self.dae3 = None self.dae4 = None self.dae5 = None self.optimizer = None self.setup_optimizer() def setup_optimizer(self): self.optimizer = optimizers.Adam() self.optimizer.setup(self.model) def dae_train(self, rng, n_epoch, batchsize, dae_num, data, n_inputs, n_hidden, corruption_level, gpu): #initialize dae = DA(rng=rng, data=data, n_inputs=n_inputs, n_hidden=n_hidden, corruption_level=corruption_level, gpu=gpu) #train print "--------DA%d training has started!--------" % dae_num dae.train_and_test(n_epoch=n_epoch, batchsize=batchsize) dae.to_cpu() # compute outputs for next dAE tmp1 = dae.compute_hidden(data[0]) tmp2 = dae.compute_hidden(data[1]) if gpu >= 0: dae.to_gpu() next_inputs = [tmp1, tmp2] return dae, next_inputs def pre_train(self, n_epoch=20, batchsize=40, sda_name="SDA"): first_inputs = self.data n_epoch1 = n_epoch batchsize1 = batchsize # initialize first dAE self.dae1, second_inputs = self.dae_train( self.rng, n_epoch=n_epoch, batchsize=batchsize, dae_num=1, data=first_inputs, n_inputs=self.n_inputs, n_hidden=self.n_hidden[0], corruption_level=self.corruption_levels[0], gpu=self.gpu) self.dae2, third_inputs = self.dae_train( self.rng, n_epoch=int(n_epoch), batchsize=batchsize, dae_num=2, data=second_inputs, n_inputs=self.n_hidden[0], n_hidden=self.n_hidden[1], corruption_level=self.corruption_levels[1], gpu=self.gpu) self.dae3, forth_inputs = self.dae_train( self.rng, n_epoch=int(n_epoch), batchsize=batchsize, dae_num=3, data=third_inputs, n_inputs=self.n_hidden[1], n_hidden=self.n_hidden[2], corruption_level=self.corruption_levels[2], gpu=self.gpu) self.dae4, fifth_inputs = self.dae_train( self.rng, n_epoch=int(n_epoch), batchsize=batchsize, dae_num=4, data=forth_inputs, n_inputs=self.n_hidden[2], n_hidden=self.n_hidden[3], corruption_level=self.corruption_levels[3], gpu=self.gpu) self.dae5, sixth_inputs = self.dae_train( self.rng, n_epoch=int(n_epoch), batchsize=batchsize, dae_num=5, data=fifth_inputs, n_inputs=self.n_hidden[3], n_hidden=self.n_hidden[4], corruption_level=self.corruption_levels[4], gpu=self.gpu) # update model parameters self.model.l1 = self.dae1.model.encoder self.model.l2 = self.dae2.model.encoder self.model.l3 = self.dae3.model.encoder self.model.l4 = self.dae4.model.encoder self.model.l5 = self.dae5.model.encoder self.setup_optimizer() model_file = "%s.model" % sda_name state_file = "%s.state" % sda_name serializers.save_hdf5(model_file, self.model) serializers.save_hdf5(state_file, self.optimizer) def forward(self, x_data, y_data, train=True, output=False): x, t = Variable(x_data), Variable(y_data) h1 = F.dropout(F.relu(self.model.l1(x)), train=train) h2 = F.dropout(F.relu(self.model.l2(h1)), train=train) h3 = F.dropout(F.relu(self.model.l3(h2)), train=train) h4 = F.dropout(F.relu(self.model.l4(h3)), train=train) h5 = F.dropout(F.relu(self.model.l5(h4)), train=train) y = F.tanh(self.model.l6(h5)) if output: return y else: return F.mean_squared_error(y, t) def fine_tune(self, n_epoch=20, batchsize=50): train_accs = [] test_accs = [] #早期終了用配列 self.save_accuracy = self.xp.tile([1000.0], 100) #ベストLOSS定義 self.best_loss = 1000.0 for epoch in xrange(1, n_epoch + 1): print 'fine tuning epoch ', epoch perm = self.rng.permutation(self.n_train) sum_loss = 0 for i in xrange(0, self.n_train, batchsize): x_batch = self.xp.asarray(self.x_train[perm[i:i + batchsize]]) y_batch = self.xp.asarray(self.y_train[perm[i:i + batchsize]]) real_batchsize = len(x_batch) self.optimizer.zero_grads() loss = self.forward(x_batch, y_batch) loss.backward() self.optimizer.update() sum_loss += float(cuda.to_cpu(loss.data)) * real_batchsize print 'fine tuning train mean loss={}'.format(sum_loss / self.n_train) train_accs.append(sum_loss / self.n_train) # evaluation sum_loss = 0 for i in xrange(0, self.n_test, batchsize): x_batch = self.xp.asarray(self.x_test[i:i + batchsize]) y_batch = self.xp.asarray(self.y_test[i:i + batchsize]) real_batchsize = len(x_batch) loss = self.forward(x_batch, y_batch, train=False) sum_loss += float(cuda.to_cpu(loss.data)) * real_batchsize print 'fine tuning test mean loss={}'.format(sum_loss / self.n_test) test_accs.append(sum_loss / self.n_test) if sum_loss < self.best_loss: self.best_loss = sum_loss self.best_epoch = epoch serializers.save_hdf5('mlp.model', self.model) print("update best loss") #早期終了? if self.xp.mean(self.save_accuracy) < sum_loss: print("early stopping done") break #早期終了用配列にsum_accuracyを追加 self.save_accuracy = self.save_accuracy[1:] append = self.xp.array([float(sum_loss)]) self.save_accuracy = self.xp.hstack((self.save_accuracy, append)) print("best_epoch: %d" % (self.best_epoch)) serializers.load_hdf5("mlp.model", self.model) return train_accs, test_accs
class QNet: # Hyper-Parameters gamma = 0.95 # Discount factor timestep_per_episode = 5000 initial_exploration = timestep_per_episode * 1 # Initial exploratoin. original: 5x10^4 replay_size = 32 # Replay (batch) size hist_size = 2 # original: 4 data_index = 0 data_flag = False loss_log = '../playground/Assets/log/' def __init__(self, use_gpu, enable_controller, cnn_input_dim, feature_dim, agent_count, other_input_dim, model): self.use_gpu = use_gpu self.num_of_actions = len(enable_controller) self.enable_controller = enable_controller self.cnn_input_dim = cnn_input_dim self.feature_dim = feature_dim self.agent_count = agent_count self.other_input_dim = other_input_dim self.data_size = self.timestep_per_episode self.loss_log_file = self.loss_log + "loss.log" self.loss_per_episode = 0 self.time_of_episode = 0 print("Initializing Q-Network...") if model == 'None': self.model = Chain( conv1=L.Convolution2D(3 * self.hist_size, 32, 4, stride=2), bn1=L.BatchNormalization(32), conv2=L.Convolution2D(32, 32, 4, stride=2), bn2=L.BatchNormalization(32), conv3=L.Convolution2D(32, 32, 4, stride=2), bn3=L.BatchNormalization(32), # conv4=L.Convolution2D(64, 64, 4, stride=2), # bn4=L.BatchNormalization(64), l1=L.Linear( self.feature_dim + self.other_input_dim * self.hist_size, 128), l2=L.Linear(128, 128), l3=L.Linear(128, 96), l4=L.Linear(96, 64), q_value=L.Linear(64, self.num_of_actions)) else: with open(model, 'rb') as i: self.model = pickle.load(i) self.data_size = 0 if self.use_gpu >= 0: self.model.to_gpu() self.optimizer = optimizers.RMSpropGraves() self.optimizer.setup(self.model) # History Data : D=[s, a, r, s_dash, end_episode_flag] self.d = [ np.zeros((self.agent_count, self.data_size, self.hist_size, 128, 128, 3), dtype=np.uint8), np.zeros((self.agent_count, self.data_size, self.hist_size, self.other_input_dim), dtype=np.uint8), np.zeros((self.agent_count, self.data_size), dtype=np.uint8), np.zeros((self.agent_count, self.data_size, 1), dtype=np.float32), np.zeros((self.agent_count, self.data_size, 1), dtype=np.bool) ] def _reshape_for_cnn(self, state, batch_size, hist_size, x, y): state_ = np.zeros((batch_size, 3 * hist_size, 128, 128), dtype=np.float32) for i in range(batch_size): if self.hist_size == 1: state_[i] = state[i][0].transpose(2, 0, 1) elif self.hist_size == 2: state_[i] = np.c_[state[i][0], state[i][1]].transpose(2, 0, 1) elif self.hist_size == 4: state_[i] = np.c_[state[i][0], state[i][1], state[i][2], state[i][3]].transpose(2, 0, 1) return state_ def forward(self, state_cnn, state_other, action, reward, state_cnn_dash, state_other_dash, episode_end): num_of_batch = state_cnn.shape[0] s_cnn = Variable(state_cnn) s_oth = Variable(state_other) s_cnn_dash = Variable(state_cnn_dash) s_oth_dash = Variable(state_other_dash) q = self.q_func(s_cnn, s_oth) # Get Q-value max_q_dash_ = self.q_func(s_cnn_dash, s_oth_dash) if self.use_gpu >= 0: tmp = list(map(np.max, max_q_dash_.data.get())) else: tmp = list(map(np.max, max_q_dash_.data)) max_q_dash = np.asanyarray(tmp, dtype=np.float32) if self.use_gpu >= 0: target = np.array(q.data.get(), dtype=np.float32) else: target = np.array(q.data, dtype=np.float32) for i in range(num_of_batch): tmp_ = reward[i] + (1 - episode_end[i]) * self.gamma * max_q_dash[i] action_index = self.action_to_index(action[i]) target[i, action_index] = tmp_ if self.use_gpu >= 0: loss = F.mean_squared_error(Variable(cuda.to_gpu(target)), q) else: loss = F.mean_squared_error(Variable(target), q) return loss, q def stock_experience(self, time, state_cnn, state_other, action, reward, state_cnn_dash, state_other_dash, episode_end_flag): for i in range(self.agent_count): self.d[0][i][self.data_index] = state_cnn[i].copy() self.d[1][i][self.data_index] = state_other[i].copy() self.d[2][i][self.data_index] = action[i].copy() self.d[3][i][self.data_index] = reward[i].copy() self.d[4][i][self.data_index] = episode_end_flag self.data_index += 1 if self.data_index >= self.data_size: self.data_index -= self.data_size self.data_flag = True def experience_replay(self, time): if self.initial_exploration < time: # Pick up replay_size number of samples from the Data replayRobotIndex = np.random.randint(0, self.agent_count, self.replay_size) if not self.data_flag: # during the first sweep of the History Data replay_index = np.random.randint(0, self.data_index, self.replay_size) else: replay_index = np.random.randint(0, self.data_size, self.replay_size) s_cnn_replay = np.ndarray(shape=(self.replay_size, self.hist_size, 128, 128, 3), dtype=np.float32) s_oth_replay = np.ndarray(shape=(self.replay_size, self.hist_size, self.other_input_dim), dtype=np.float32) a_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.uint8) r_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.float32) s_cnn_dash_replay = np.ndarray(shape=(self.replay_size, self.hist_size, 128, 128, 3), dtype=np.float32) s_oth_dash_replay = np.ndarray(shape=(self.replay_size, self.hist_size, self.other_input_dim), dtype=np.float32) episode_end_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.bool) for i in range(self.replay_size): s_cnn_replay[i] = np.asarray( (self.d[0][replayRobotIndex[i]][replay_index[i]]), dtype=np.float32) s_oth_replay[i] = np.asarray( (self.d[1][replayRobotIndex[i]][replay_index[i]]), dtype=np.float32) a_replay[i] = self.d[2][replayRobotIndex[i]][replay_index[i]] r_replay[i] = self.d[3][replayRobotIndex[i]][replay_index[i]] if (replay_index[i] + 1 >= self.data_size): s_cnn_dash_replay[i] = np.array( (self.d[0][replayRobotIndex[i]][replay_index[i] + 1 - self.data_size]), dtype=np.float32) s_oth_dash_replay[i] = np.array( (self.d[1][replayRobotIndex[i]][replay_index[i] + 1 - self.data_size]), dtype=np.float32) else: s_cnn_dash_replay[i] = np.array( (self.d[0][replayRobotIndex[i]][replay_index[i] + 1]), dtype=np.float32) s_oth_dash_replay[i] = np.array( (self.d[1][replayRobotIndex[i]][replay_index[i] + 1]), dtype=np.float32) episode_end_replay[i] = self.d[4][replayRobotIndex[i]][ replay_index[i]] s_cnn_replay = self._reshape_for_cnn(s_cnn_replay, self.replay_size, self.hist_size, 128, 128) s_cnn_dash_replay = self._reshape_for_cnn(s_cnn_dash_replay, self.replay_size, self.hist_size, 128, 128) s_cnn_replay /= 255.0 s_oth_replay /= 255.0 s_cnn_dash_replay /= 255.0 s_oth_dash_replay /= 255.0 if self.use_gpu >= 0: s_cnn_replay = cuda.to_gpu(s_cnn_replay) s_oth_replay = cuda.to_gpu(s_oth_replay) s_cnn_dash_replay = cuda.to_gpu(s_cnn_dash_replay) s_oth_dash_replay = cuda.to_gpu(s_oth_dash_replay) # Gradient-based update loss, _ = self.forward(s_cnn_replay, s_oth_replay, a_replay, r_replay, s_cnn_dash_replay, s_oth_dash_replay, episode_end_replay) send_loss = loss.data with open(self.loss_log_file, 'a') as the_file: the_file.write(str(time) + "," + str(send_loss) + "\n") self.loss_per_episode += loss.data self.time_of_episode += 1 self.model.zerograds() loss.backward() self.optimizer.update() def q_func(self, state_cnn, state_other): if self.use_gpu >= 0: num_of_batch = state_cnn.data.get().shape[0] else: num_of_batch = state_cnn.data.shape[0] h1 = F.tanh(self.model.bn1(self.model.conv1(state_cnn))) h2 = F.tanh(self.model.bn2(self.model.conv2(h1))) h3 = F.tanh(self.model.bn3(self.model.conv3(h2))) # h4 = F.tanh(self.model.bn4(self.model.conv4(h3))) # h5 = F.tanh(self.model.bn5(self.model.conv5(h4))) h4_ = F.concat( (F.reshape(h3, (num_of_batch, self.feature_dim)), F.reshape(state_other, (num_of_batch, self.other_input_dim * self.hist_size))), axis=1) h6 = F.relu(self.model.l1(h4_)) h7 = F.relu(self.model.l2(h6)) h8 = F.relu(self.model.l3(h7)) h9 = F.relu(self.model.l4(h8)) q = self.model.q_value(h9) return q def e_greedy(self, state_cnn, state_other, epsilon, reward): s_cnn = Variable(state_cnn) s_oth = Variable(state_other) q = self.q_func(s_cnn, s_oth) q = q.data if self.use_gpu >= 0: q_ = q.get() else: q_ = q index_action = np.zeros((self.agent_count), dtype=np.uint8) print(("agent"), end=' ') for i in range(self.agent_count): if np.random.rand() < epsilon: index_action[i] = np.random.randint(0, self.num_of_actions) print(("[%02d] Random(%2d)reward(%06.2f)" % (i, index_action[i], reward[i])), end=' ') else: index_action[i] = np.argmax(q_[i]) print(("[%02d]!Greedy(%2d)reward(%06.2f)" % (i, index_action[i], reward[i])), end=' ') if i % 5 == 4: print(("\n "), end=' ') del q_ return self.index_to_action(index_action), q def index_to_action(self, index_of_action): index = np.zeros((self.agent_count), dtype=np.uint8) for i in range(self.agent_count): index[i] = self.enable_controller[index_of_action[i]] return index def action_to_index(self, action): return self.enable_controller.index(action)