class FaceNet():
def __init__(self):
self.model = Chain(conv1=L.Convolution2D(3, 20, 3, 1, 1),
conv2=L.Convolution2D(20, 20, 3, 1, 1),
conv3=L.Convolution2D(20, 40, 3, 1, 1),
conv4=L.Convolution2D(40, 40, 3, 1, 1),
linear1=L.Linear(None, 100),
linear2=L.Linear(100, 4))
self.optimizer = optimizers.Adam()
self.optimizer.setup(self.model)
def foward(self, x):
out = self.model.conv1(x)
out = F.elu(out)
out = self.model.conv2(out)
out = F.max_pooling_2d(out, 2)
out = F.elu(out)
out = self.model.conv3(out)
out = F.elu(out)
out = self.model.conv4(out)
out = F.elu(out)
out = F.average_pooling_2d(out, 6)
out = F.dropout(out)
out = self.model.linear1(out)
out = F.elu(out)
out = F.dropout(out)
out = self.model.linear2(out)
return out
def predict(self, X, step=100):
with chainer.using_config('train', False):
with chainer.no_backprop_mode():
output = []
for i in range(0, len(X), step):
x = Variable(X[i:i + step])
output.append(self.foward(x).data)
return np.vstack(output)
def score(self, X, Y, step=100):
predicted = self.predict(X, step)
score = F.r2_score(predicted, Y).data
return score
def fit(self, X, Y, batchsize=100, n_epoch=10):
with chainer.using_config('train', True):
learning_curve = []
for epoch in range(n_epoch):
print('epoch ', epoch)
index = np.random.permutation(len(X))
for i in range(0, len(index), batchsize):
self.model.cleargrads()
print(i)
x = X[index[i:i + batchsize]]
y = Y[index[i:i + batchsize]]
#augment(x, y)
x = Variable(x)
y = Variable(y)
output = self.foward(x)
loss = F.mean_squared_error(y, output)
loss.backward()
learning_curve.append(float(loss.data))
self.optimizer.update()
return learning_curve
class QNet:
# Hyper-Parameters
gamma = 0.95 # Discount factor
timestep_per_episode = 5000
initial_exploration = timestep_per_episode * 1 # Initial exploratoin. original: 5x10^4
replay_size = 32 # Replay (batch) size
hist_size = 2 # original: 4
data_index = 0
data_flag = False
loss_log = '../playground/Assets/log/'
def __init__(self, use_gpu, enable_controller, cnn_input_dim, feature_dim,
agent_count, other_input_dim, model):
self.use_gpu = use_gpu
self.num_of_actions = len(enable_controller)
self.enable_controller = enable_controller
self.cnn_input_dim = cnn_input_dim
self.feature_dim = feature_dim
self.agent_count = agent_count
self.other_input_dim = other_input_dim
self.data_size = self.timestep_per_episode
self.loss_log_file = self.loss_log + "loss.log"
self.loss_per_episode = 0
self.time_of_episode = 0
print("Initializing Q-Network...")
if model == 'None':
self.model = Chain(
conv1=L.Convolution2D(3 * self.hist_size, 32, 4, stride=2),
bn1=L.BatchNormalization(32),
conv2=L.Convolution2D(32, 32, 4, stride=2),
bn2=L.BatchNormalization(32),
conv3=L.Convolution2D(32, 32, 4, stride=2),
bn3=L.BatchNormalization(32),
# conv4=L.Convolution2D(64, 64, 4, stride=2),
# bn4=L.BatchNormalization(64),
l1=L.Linear(
self.feature_dim + self.other_input_dim * self.hist_size,
128),
l2=L.Linear(128, 128),
l3=L.Linear(128, 96),
l4=L.Linear(96, 64),
q_value=L.Linear(64, self.num_of_actions))
else:
with open(model, 'rb') as i:
self.model = pickle.load(i)
self.data_size = 0
if self.use_gpu >= 0:
self.model.to_gpu()
self.optimizer = optimizers.RMSpropGraves()
self.optimizer.setup(self.model)
# History Data : D=[s, a, r, s_dash, end_episode_flag]
self.d = [
np.zeros((self.agent_count, self.data_size, self.hist_size, 128,
128, 3),
dtype=np.uint8),
np.zeros((self.agent_count, self.data_size, self.hist_size,
self.other_input_dim),
dtype=np.uint8),
np.zeros((self.agent_count, self.data_size), dtype=np.uint8),
np.zeros((self.agent_count, self.data_size, 1), dtype=np.float32),
np.zeros((self.agent_count, self.data_size, 1), dtype=np.bool)
]
def _reshape_for_cnn(self, state, batch_size, hist_size, x, y):
state_ = np.zeros((batch_size, 3 * hist_size, 128, 128),
dtype=np.float32)
for i in range(batch_size):
if self.hist_size == 1:
state_[i] = state[i][0].transpose(2, 0, 1)
elif self.hist_size == 2:
state_[i] = np.c_[state[i][0], state[i][1]].transpose(2, 0, 1)
elif self.hist_size == 4:
state_[i] = np.c_[state[i][0], state[i][1], state[i][2],
state[i][3]].transpose(2, 0, 1)
return state_
def forward(self, state_cnn, state_other, action, reward, state_cnn_dash,
state_other_dash, episode_end):
num_of_batch = state_cnn.shape[0]
s_cnn = Variable(state_cnn)
s_oth = Variable(state_other)
s_cnn_dash = Variable(state_cnn_dash)
s_oth_dash = Variable(state_other_dash)
q = self.q_func(s_cnn, s_oth) # Get Q-value
max_q_dash_ = self.q_func(s_cnn_dash, s_oth_dash)
if self.use_gpu >= 0:
tmp = list(map(np.max, max_q_dash_.data.get()))
else:
tmp = list(map(np.max, max_q_dash_.data))
max_q_dash = np.asanyarray(tmp, dtype=np.float32)
if self.use_gpu >= 0:
target = np.array(q.data.get(), dtype=np.float32)
else:
target = np.array(q.data, dtype=np.float32)
for i in range(num_of_batch):
tmp_ = reward[i] + (1 -
episode_end[i]) * self.gamma * max_q_dash[i]
action_index = self.action_to_index(action[i])
target[i, action_index] = tmp_
if self.use_gpu >= 0:
loss = F.mean_squared_error(Variable(cuda.to_gpu(target)), q)
else:
loss = F.mean_squared_error(Variable(target), q)
return loss, q
def stock_experience(self, time, state_cnn, state_other, action, reward,
state_cnn_dash, state_other_dash, episode_end_flag):
for i in range(self.agent_count):
self.d[0][i][self.data_index] = state_cnn[i].copy()
self.d[1][i][self.data_index] = state_other[i].copy()
self.d[2][i][self.data_index] = action[i].copy()
self.d[3][i][self.data_index] = reward[i].copy()
self.d[4][i][self.data_index] = episode_end_flag
self.data_index += 1
if self.data_index >= self.data_size:
self.data_index -= self.data_size
self.data_flag = True
def experience_replay(self, time):
if self.initial_exploration < time:
# Pick up replay_size number of samples from the Data
replayRobotIndex = np.random.randint(0, self.agent_count,
self.replay_size)
if not self.data_flag: # during the first sweep of the History Data
replay_index = np.random.randint(0, self.data_index,
self.replay_size)
else:
replay_index = np.random.randint(0, self.data_size,
self.replay_size)
s_cnn_replay = np.ndarray(shape=(self.replay_size, self.hist_size,
128, 128, 3),
dtype=np.float32)
s_oth_replay = np.ndarray(shape=(self.replay_size, self.hist_size,
self.other_input_dim),
dtype=np.float32)
a_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.uint8)
r_replay = np.ndarray(shape=(self.replay_size, 1),
dtype=np.float32)
s_cnn_dash_replay = np.ndarray(shape=(self.replay_size,
self.hist_size, 128, 128, 3),
dtype=np.float32)
s_oth_dash_replay = np.ndarray(shape=(self.replay_size,
self.hist_size,
self.other_input_dim),
dtype=np.float32)
episode_end_replay = np.ndarray(shape=(self.replay_size, 1),
dtype=np.bool)
for i in range(self.replay_size):
s_cnn_replay[i] = np.asarray(
(self.d[0][replayRobotIndex[i]][replay_index[i]]),
dtype=np.float32)
s_oth_replay[i] = np.asarray(
(self.d[1][replayRobotIndex[i]][replay_index[i]]),
dtype=np.float32)
a_replay[i] = self.d[2][replayRobotIndex[i]][replay_index[i]]
r_replay[i] = self.d[3][replayRobotIndex[i]][replay_index[i]]
if (replay_index[i] + 1 >= self.data_size):
s_cnn_dash_replay[i] = np.array(
(self.d[0][replayRobotIndex[i]][replay_index[i] + 1 -
self.data_size]),
dtype=np.float32)
s_oth_dash_replay[i] = np.array(
(self.d[1][replayRobotIndex[i]][replay_index[i] + 1 -
self.data_size]),
dtype=np.float32)
else:
s_cnn_dash_replay[i] = np.array(
(self.d[0][replayRobotIndex[i]][replay_index[i] + 1]),
dtype=np.float32)
s_oth_dash_replay[i] = np.array(
(self.d[1][replayRobotIndex[i]][replay_index[i] + 1]),
dtype=np.float32)
episode_end_replay[i] = self.d[4][replayRobotIndex[i]][
replay_index[i]]
s_cnn_replay = self._reshape_for_cnn(s_cnn_replay,
self.replay_size,
self.hist_size, 128, 128)
s_cnn_dash_replay = self._reshape_for_cnn(s_cnn_dash_replay,
self.replay_size,
self.hist_size, 128, 128)
s_cnn_replay /= 255.0
s_oth_replay /= 255.0
s_cnn_dash_replay /= 255.0
s_oth_dash_replay /= 255.0
if self.use_gpu >= 0:
s_cnn_replay = cuda.to_gpu(s_cnn_replay)
s_oth_replay = cuda.to_gpu(s_oth_replay)
s_cnn_dash_replay = cuda.to_gpu(s_cnn_dash_replay)
s_oth_dash_replay = cuda.to_gpu(s_oth_dash_replay)
# Gradient-based update
loss, _ = self.forward(s_cnn_replay, s_oth_replay, a_replay,
r_replay, s_cnn_dash_replay,
s_oth_dash_replay, episode_end_replay)
send_loss = loss.data
with open(self.loss_log_file, 'a') as the_file:
the_file.write(str(time) + "," + str(send_loss) + "\n")
self.loss_per_episode += loss.data
self.time_of_episode += 1
self.model.zerograds()
loss.backward()
self.optimizer.update()
def q_func(self, state_cnn, state_other):
if self.use_gpu >= 0:
num_of_batch = state_cnn.data.get().shape[0]
else:
num_of_batch = state_cnn.data.shape[0]
h1 = F.tanh(self.model.bn1(self.model.conv1(state_cnn)))
h2 = F.tanh(self.model.bn2(self.model.conv2(h1)))
h3 = F.tanh(self.model.bn3(self.model.conv3(h2)))
# h4 = F.tanh(self.model.bn4(self.model.conv4(h3)))
# h5 = F.tanh(self.model.bn5(self.model.conv5(h4)))
h4_ = F.concat(
(F.reshape(h3, (num_of_batch, self.feature_dim)),
F.reshape(state_other,
(num_of_batch, self.other_input_dim * self.hist_size))),
axis=1)
h6 = F.relu(self.model.l1(h4_))
h7 = F.relu(self.model.l2(h6))
h8 = F.relu(self.model.l3(h7))
h9 = F.relu(self.model.l4(h8))
q = self.model.q_value(h9)
return q
def e_greedy(self, state_cnn, state_other, epsilon, reward):
s_cnn = Variable(state_cnn)
s_oth = Variable(state_other)
q = self.q_func(s_cnn, s_oth)
q = q.data
if self.use_gpu >= 0:
q_ = q.get()
else:
q_ = q
index_action = np.zeros((self.agent_count), dtype=np.uint8)
print(("agent"), end=' ')
for i in range(self.agent_count):
if np.random.rand() < epsilon:
index_action[i] = np.random.randint(0, self.num_of_actions)
print(("[%02d] Random(%2d)reward(%06.2f)" %
(i, index_action[i], reward[i])),
end=' ')
else:
index_action[i] = np.argmax(q_[i])
print(("[%02d]!Greedy(%2d)reward(%06.2f)" %
(i, index_action[i], reward[i])),
end=' ')
if i % 5 == 4:
print(("\n "), end=' ')
del q_
return self.index_to_action(index_action), q
def index_to_action(self, index_of_action):
index = np.zeros((self.agent_count), dtype=np.uint8)
for i in range(self.agent_count):
index[i] = self.enable_controller[index_of_action[i]]
return index
def action_to_index(self, action):
return self.enable_controller.index(action)