class FaceNet(): def __init__(self): self.model = Chain(conv1=L.Convolution2D(3, 20, 3, 1, 1), conv2=L.Convolution2D(20, 20, 3, 1, 1), conv3=L.Convolution2D(20, 40, 3, 1, 1), conv4=L.Convolution2D(40, 40, 3, 1, 1), linear1=L.Linear(None, 100), linear2=L.Linear(100, 4)) self.optimizer = optimizers.Adam() self.optimizer.setup(self.model) def foward(self, x): out = self.model.conv1(x) out = F.elu(out) out = self.model.conv2(out) out = F.max_pooling_2d(out, 2) out = F.elu(out) out = self.model.conv3(out) out = F.elu(out) out = self.model.conv4(out) out = F.elu(out) out = F.average_pooling_2d(out, 6) out = F.dropout(out) out = self.model.linear1(out) out = F.elu(out) out = F.dropout(out) out = self.model.linear2(out) return out def predict(self, X, step=100): with chainer.using_config('train', False): with chainer.no_backprop_mode(): output = [] for i in range(0, len(X), step): x = Variable(X[i:i + step]) output.append(self.foward(x).data) return np.vstack(output) def score(self, X, Y, step=100): predicted = self.predict(X, step) score = F.r2_score(predicted, Y).data return score def fit(self, X, Y, batchsize=100, n_epoch=10): with chainer.using_config('train', True): learning_curve = [] for epoch in range(n_epoch): print('epoch ', epoch) index = np.random.permutation(len(X)) for i in range(0, len(index), batchsize): self.model.cleargrads() print(i) x = X[index[i:i + batchsize]] y = Y[index[i:i + batchsize]] #augment(x, y) x = Variable(x) y = Variable(y) output = self.foward(x) loss = F.mean_squared_error(y, output) loss.backward() learning_curve.append(float(loss.data)) self.optimizer.update() return learning_curve
class QNet: # Hyper-Parameters gamma = 0.95 # Discount factor timestep_per_episode = 5000 initial_exploration = timestep_per_episode * 1 # Initial exploratoin. original: 5x10^4 replay_size = 32 # Replay (batch) size hist_size = 2 # original: 4 data_index = 0 data_flag = False loss_log = '../playground/Assets/log/' def __init__(self, use_gpu, enable_controller, cnn_input_dim, feature_dim, agent_count, other_input_dim, model): self.use_gpu = use_gpu self.num_of_actions = len(enable_controller) self.enable_controller = enable_controller self.cnn_input_dim = cnn_input_dim self.feature_dim = feature_dim self.agent_count = agent_count self.other_input_dim = other_input_dim self.data_size = self.timestep_per_episode self.loss_log_file = self.loss_log + "loss.log" self.loss_per_episode = 0 self.time_of_episode = 0 print("Initializing Q-Network...") if model == 'None': self.model = Chain( conv1=L.Convolution2D(3 * self.hist_size, 32, 4, stride=2), bn1=L.BatchNormalization(32), conv2=L.Convolution2D(32, 32, 4, stride=2), bn2=L.BatchNormalization(32), conv3=L.Convolution2D(32, 32, 4, stride=2), bn3=L.BatchNormalization(32), # conv4=L.Convolution2D(64, 64, 4, stride=2), # bn4=L.BatchNormalization(64), l1=L.Linear( self.feature_dim + self.other_input_dim * self.hist_size, 128), l2=L.Linear(128, 128), l3=L.Linear(128, 96), l4=L.Linear(96, 64), q_value=L.Linear(64, self.num_of_actions)) else: with open(model, 'rb') as i: self.model = pickle.load(i) self.data_size = 0 if self.use_gpu >= 0: self.model.to_gpu() self.optimizer = optimizers.RMSpropGraves() self.optimizer.setup(self.model) # History Data : D=[s, a, r, s_dash, end_episode_flag] self.d = [ np.zeros((self.agent_count, self.data_size, self.hist_size, 128, 128, 3), dtype=np.uint8), np.zeros((self.agent_count, self.data_size, self.hist_size, self.other_input_dim), dtype=np.uint8), np.zeros((self.agent_count, self.data_size), dtype=np.uint8), np.zeros((self.agent_count, self.data_size, 1), dtype=np.float32), np.zeros((self.agent_count, self.data_size, 1), dtype=np.bool) ] def _reshape_for_cnn(self, state, batch_size, hist_size, x, y): state_ = np.zeros((batch_size, 3 * hist_size, 128, 128), dtype=np.float32) for i in range(batch_size): if self.hist_size == 1: state_[i] = state[i][0].transpose(2, 0, 1) elif self.hist_size == 2: state_[i] = np.c_[state[i][0], state[i][1]].transpose(2, 0, 1) elif self.hist_size == 4: state_[i] = np.c_[state[i][0], state[i][1], state[i][2], state[i][3]].transpose(2, 0, 1) return state_ def forward(self, state_cnn, state_other, action, reward, state_cnn_dash, state_other_dash, episode_end): num_of_batch = state_cnn.shape[0] s_cnn = Variable(state_cnn) s_oth = Variable(state_other) s_cnn_dash = Variable(state_cnn_dash) s_oth_dash = Variable(state_other_dash) q = self.q_func(s_cnn, s_oth) # Get Q-value max_q_dash_ = self.q_func(s_cnn_dash, s_oth_dash) if self.use_gpu >= 0: tmp = list(map(np.max, max_q_dash_.data.get())) else: tmp = list(map(np.max, max_q_dash_.data)) max_q_dash = np.asanyarray(tmp, dtype=np.float32) if self.use_gpu >= 0: target = np.array(q.data.get(), dtype=np.float32) else: target = np.array(q.data, dtype=np.float32) for i in range(num_of_batch): tmp_ = reward[i] + (1 - episode_end[i]) * self.gamma * max_q_dash[i] action_index = self.action_to_index(action[i]) target[i, action_index] = tmp_ if self.use_gpu >= 0: loss = F.mean_squared_error(Variable(cuda.to_gpu(target)), q) else: loss = F.mean_squared_error(Variable(target), q) return loss, q def stock_experience(self, time, state_cnn, state_other, action, reward, state_cnn_dash, state_other_dash, episode_end_flag): for i in range(self.agent_count): self.d[0][i][self.data_index] = state_cnn[i].copy() self.d[1][i][self.data_index] = state_other[i].copy() self.d[2][i][self.data_index] = action[i].copy() self.d[3][i][self.data_index] = reward[i].copy() self.d[4][i][self.data_index] = episode_end_flag self.data_index += 1 if self.data_index >= self.data_size: self.data_index -= self.data_size self.data_flag = True def experience_replay(self, time): if self.initial_exploration < time: # Pick up replay_size number of samples from the Data replayRobotIndex = np.random.randint(0, self.agent_count, self.replay_size) if not self.data_flag: # during the first sweep of the History Data replay_index = np.random.randint(0, self.data_index, self.replay_size) else: replay_index = np.random.randint(0, self.data_size, self.replay_size) s_cnn_replay = np.ndarray(shape=(self.replay_size, self.hist_size, 128, 128, 3), dtype=np.float32) s_oth_replay = np.ndarray(shape=(self.replay_size, self.hist_size, self.other_input_dim), dtype=np.float32) a_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.uint8) r_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.float32) s_cnn_dash_replay = np.ndarray(shape=(self.replay_size, self.hist_size, 128, 128, 3), dtype=np.float32) s_oth_dash_replay = np.ndarray(shape=(self.replay_size, self.hist_size, self.other_input_dim), dtype=np.float32) episode_end_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.bool) for i in range(self.replay_size): s_cnn_replay[i] = np.asarray( (self.d[0][replayRobotIndex[i]][replay_index[i]]), dtype=np.float32) s_oth_replay[i] = np.asarray( (self.d[1][replayRobotIndex[i]][replay_index[i]]), dtype=np.float32) a_replay[i] = self.d[2][replayRobotIndex[i]][replay_index[i]] r_replay[i] = self.d[3][replayRobotIndex[i]][replay_index[i]] if (replay_index[i] + 1 >= self.data_size): s_cnn_dash_replay[i] = np.array( (self.d[0][replayRobotIndex[i]][replay_index[i] + 1 - self.data_size]), dtype=np.float32) s_oth_dash_replay[i] = np.array( (self.d[1][replayRobotIndex[i]][replay_index[i] + 1 - self.data_size]), dtype=np.float32) else: s_cnn_dash_replay[i] = np.array( (self.d[0][replayRobotIndex[i]][replay_index[i] + 1]), dtype=np.float32) s_oth_dash_replay[i] = np.array( (self.d[1][replayRobotIndex[i]][replay_index[i] + 1]), dtype=np.float32) episode_end_replay[i] = self.d[4][replayRobotIndex[i]][ replay_index[i]] s_cnn_replay = self._reshape_for_cnn(s_cnn_replay, self.replay_size, self.hist_size, 128, 128) s_cnn_dash_replay = self._reshape_for_cnn(s_cnn_dash_replay, self.replay_size, self.hist_size, 128, 128) s_cnn_replay /= 255.0 s_oth_replay /= 255.0 s_cnn_dash_replay /= 255.0 s_oth_dash_replay /= 255.0 if self.use_gpu >= 0: s_cnn_replay = cuda.to_gpu(s_cnn_replay) s_oth_replay = cuda.to_gpu(s_oth_replay) s_cnn_dash_replay = cuda.to_gpu(s_cnn_dash_replay) s_oth_dash_replay = cuda.to_gpu(s_oth_dash_replay) # Gradient-based update loss, _ = self.forward(s_cnn_replay, s_oth_replay, a_replay, r_replay, s_cnn_dash_replay, s_oth_dash_replay, episode_end_replay) send_loss = loss.data with open(self.loss_log_file, 'a') as the_file: the_file.write(str(time) + "," + str(send_loss) + "\n") self.loss_per_episode += loss.data self.time_of_episode += 1 self.model.zerograds() loss.backward() self.optimizer.update() def q_func(self, state_cnn, state_other): if self.use_gpu >= 0: num_of_batch = state_cnn.data.get().shape[0] else: num_of_batch = state_cnn.data.shape[0] h1 = F.tanh(self.model.bn1(self.model.conv1(state_cnn))) h2 = F.tanh(self.model.bn2(self.model.conv2(h1))) h3 = F.tanh(self.model.bn3(self.model.conv3(h2))) # h4 = F.tanh(self.model.bn4(self.model.conv4(h3))) # h5 = F.tanh(self.model.bn5(self.model.conv5(h4))) h4_ = F.concat( (F.reshape(h3, (num_of_batch, self.feature_dim)), F.reshape(state_other, (num_of_batch, self.other_input_dim * self.hist_size))), axis=1) h6 = F.relu(self.model.l1(h4_)) h7 = F.relu(self.model.l2(h6)) h8 = F.relu(self.model.l3(h7)) h9 = F.relu(self.model.l4(h8)) q = self.model.q_value(h9) return q def e_greedy(self, state_cnn, state_other, epsilon, reward): s_cnn = Variable(state_cnn) s_oth = Variable(state_other) q = self.q_func(s_cnn, s_oth) q = q.data if self.use_gpu >= 0: q_ = q.get() else: q_ = q index_action = np.zeros((self.agent_count), dtype=np.uint8) print(("agent"), end=' ') for i in range(self.agent_count): if np.random.rand() < epsilon: index_action[i] = np.random.randint(0, self.num_of_actions) print(("[%02d] Random(%2d)reward(%06.2f)" % (i, index_action[i], reward[i])), end=' ') else: index_action[i] = np.argmax(q_[i]) print(("[%02d]!Greedy(%2d)reward(%06.2f)" % (i, index_action[i], reward[i])), end=' ') if i % 5 == 4: print(("\n "), end=' ') del q_ return self.index_to_action(index_action), q def index_to_action(self, index_of_action): index = np.zeros((self.agent_count), dtype=np.uint8) for i in range(self.agent_count): index[i] = self.enable_controller[index_of_action[i]] return index def action_to_index(self, action): return self.enable_controller.index(action)