class ChainerDQNclass:
STATE_FRAMES = 4 # number of frames to store in the state
def __init__(self):
self.num_of_actions = 4
print "Initializing DQN..."
print "Model Building"
self.model = Chain(l1=links.Convolution2D(self.STATE_FRAMES,
32,
ksize=8,
stride=4,
nobias=False,
wscale=0.01),
l2=links.Convolution2D(32,
64,
ksize=4,
stride=2,
nobias=False,
wscale=0.01),
l3=links.Convolution2D(64,
64,
ksize=3,
stride=1,
nobias=False,
wscale=0.01),
l4=links.Linear(3136, 512, wscale=0.01),
q_value=links.Linear(512,
self.num_of_actions)).to_gpu()
self.model_target = copy.deepcopy(self.model)
# self.optimizer = optimizers.Adam(alpha=1e-6)
# self.optimizer.use_cleargrads()
# self.optimizer.setup(self.model)
def Q_func(self, state):
h1 = funcitons.relu(self.model.l1(state)) # scale inputs in [0.0 1.0]
h2 = funcitons.relu(self.model.l2(h1))
h3 = funcitons.relu(self.model.l3(h2))
h4 = funcitons.relu(self.model.l4(h3))
Q = self.model.q_value(h4)
return Q
def Q_func_target(self, state):
h1 = funcitons.relu(
self.model_target.l1(state)) # scale inputs in [0.0 1.0]
h2 = funcitons.relu(self.model_target.l2(h1))
h3 = funcitons.relu(self.model_target.l3(h2))
h4 = funcitons.relu(self.model_target.l4(h3))
Q = self.model_target.q_value(h4)
return Q
def target_model_update(self):
self.model_target = copy.deepcopy(self.model)
plt.tick_params(labelbottom="off")
plt.tick_params(labelleft="off")
# ======================================================================
# AutoEncoderの再構成画像を描画
plt.figure(figsize=(20, 20))
num = 10
cnt = 0
ans_list = []
pred_list = []
for idx in np.random.permutation(N_test)[:num]:
with using_config('train', False):
xxx = x_test[idx].astype(np.float32)
h1 = F.dropout(F.relu(model_ae.l1(Variable(xxx.reshape(1, n_dim)))))
h2 = F.dropout(F.relu(model_ae.l2(h1)))
y = model_ae.l3(h2)
cnt += 1
ans_list.append(x_test[idx])
pred_list.append(y)
cnt = 0
for i in range(int(num / 10)):
for j in range(10):
img_no = i * 10 + j
pos = (2 * i) * 10 + j
draw_digit_ae(ans_list[img_no], pos + 1, 20, 10, "ans")
for j in range(10):
img_no = i * 10 + j
pos = (2 * i + 1) * 10 + j
class SDA:
def __init__(self,
rng,
data,
target,
n_inputs=784,
n_hidden=[784, 784, 784, 784, 784],
n_outputs=1,
corruption_levels=[0.1, 0.1, 0.1, 0.1, 0.1],
gpu=-1):
self.model = Chain(l1=L.Linear(n_inputs, n_hidden[0]),
l2=L.Linear(n_hidden[0], n_hidden[1]),
l3=L.Linear(n_hidden[1], n_hidden[2]),
l4=L.Linear(n_hidden[2], n_hidden[3]),
l5=L.Linear(n_hidden[3], n_hidden[4]),
l6=L.Linear(n_hidden[4], n_outputs))
if gpu >= 0:
self.model.to_gpu()
self.xp = cuda.cupy
else:
self.xp = np
self.rng = rng
self.gpu = gpu
self.data = data
self.target = target
self.x_train, self.x_test = data
self.y_train, self.y_test = target
self.n_train = len(self.y_train)
self.n_test = len(self.y_test)
self.corruption_levels = corruption_levels
self.n_inputs = n_inputs
self.n_hidden = n_hidden
self.n_outputs = n_outputs
self.hidden_size = len(n_hidden)
self.dae1 = None
self.dae2 = None
self.dae3 = None
self.dae4 = None
self.dae5 = None
self.optimizer = None
self.setup_optimizer()
def setup_optimizer(self):
self.optimizer = optimizers.Adam()
self.optimizer.setup(self.model)
def dae_train(self, rng, n_epoch, batchsize, dae_num, data, n_inputs,
n_hidden, corruption_level, gpu):
#initialize
dae = DA(rng=rng,
data=data,
n_inputs=n_inputs,
n_hidden=n_hidden,
corruption_level=corruption_level,
gpu=gpu)
#train
print "--------DA%d training has started!--------" % dae_num
dae.train_and_test(n_epoch=n_epoch, batchsize=batchsize)
dae.to_cpu()
# compute outputs for next dAE
tmp1 = dae.compute_hidden(data[0])
tmp2 = dae.compute_hidden(data[1])
if gpu >= 0:
dae.to_gpu()
next_inputs = [tmp1, tmp2]
return dae, next_inputs
def pre_train(self, n_epoch=20, batchsize=40, sda_name="SDA"):
first_inputs = self.data
n_epoch1 = n_epoch
batchsize1 = batchsize
# initialize first dAE
self.dae1, second_inputs = self.dae_train(
self.rng,
n_epoch=n_epoch,
batchsize=batchsize,
dae_num=1,
data=first_inputs,
n_inputs=self.n_inputs,
n_hidden=self.n_hidden[0],
corruption_level=self.corruption_levels[0],
gpu=self.gpu)
self.dae2, third_inputs = self.dae_train(
self.rng,
n_epoch=int(n_epoch),
batchsize=batchsize,
dae_num=2,
data=second_inputs,
n_inputs=self.n_hidden[0],
n_hidden=self.n_hidden[1],
corruption_level=self.corruption_levels[1],
gpu=self.gpu)
self.dae3, forth_inputs = self.dae_train(
self.rng,
n_epoch=int(n_epoch),
batchsize=batchsize,
dae_num=3,
data=third_inputs,
n_inputs=self.n_hidden[1],
n_hidden=self.n_hidden[2],
corruption_level=self.corruption_levels[2],
gpu=self.gpu)
self.dae4, fifth_inputs = self.dae_train(
self.rng,
n_epoch=int(n_epoch),
batchsize=batchsize,
dae_num=4,
data=forth_inputs,
n_inputs=self.n_hidden[2],
n_hidden=self.n_hidden[3],
corruption_level=self.corruption_levels[3],
gpu=self.gpu)
self.dae5, sixth_inputs = self.dae_train(
self.rng,
n_epoch=int(n_epoch),
batchsize=batchsize,
dae_num=5,
data=fifth_inputs,
n_inputs=self.n_hidden[3],
n_hidden=self.n_hidden[4],
corruption_level=self.corruption_levels[4],
gpu=self.gpu)
# update model parameters
self.model.l1 = self.dae1.model.encoder
self.model.l2 = self.dae2.model.encoder
self.model.l3 = self.dae3.model.encoder
self.model.l4 = self.dae4.model.encoder
self.model.l5 = self.dae5.model.encoder
self.setup_optimizer()
model_file = "%s.model" % sda_name
state_file = "%s.state" % sda_name
serializers.save_hdf5(model_file, self.model)
serializers.save_hdf5(state_file, self.optimizer)
def forward(self, x_data, y_data, train=True, output=False):
x, t = Variable(x_data), Variable(y_data)
h1 = F.dropout(F.relu(self.model.l1(x)), train=train)
h2 = F.dropout(F.relu(self.model.l2(h1)), train=train)
h3 = F.dropout(F.relu(self.model.l3(h2)), train=train)
h4 = F.dropout(F.relu(self.model.l4(h3)), train=train)
h5 = F.dropout(F.relu(self.model.l5(h4)), train=train)
y = F.tanh(self.model.l6(h5))
if output:
return y
else:
return F.mean_squared_error(y, t)
def fine_tune(self, n_epoch=20, batchsize=50):
train_accs = []
test_accs = []
#早期終了用配列
self.save_accuracy = self.xp.tile([1000.0], 100)
#ベストLOSS定義
self.best_loss = 1000.0
for epoch in xrange(1, n_epoch + 1):
print 'fine tuning epoch ', epoch
perm = self.rng.permutation(self.n_train)
sum_loss = 0
for i in xrange(0, self.n_train, batchsize):
x_batch = self.xp.asarray(self.x_train[perm[i:i + batchsize]])
y_batch = self.xp.asarray(self.y_train[perm[i:i + batchsize]])
real_batchsize = len(x_batch)
self.optimizer.zero_grads()
loss = self.forward(x_batch, y_batch)
loss.backward()
self.optimizer.update()
sum_loss += float(cuda.to_cpu(loss.data)) * real_batchsize
print 'fine tuning train mean loss={}'.format(sum_loss /
self.n_train)
train_accs.append(sum_loss / self.n_train)
# evaluation
sum_loss = 0
for i in xrange(0, self.n_test, batchsize):
x_batch = self.xp.asarray(self.x_test[i:i + batchsize])
y_batch = self.xp.asarray(self.y_test[i:i + batchsize])
real_batchsize = len(x_batch)
loss = self.forward(x_batch, y_batch, train=False)
sum_loss += float(cuda.to_cpu(loss.data)) * real_batchsize
print 'fine tuning test mean loss={}'.format(sum_loss /
self.n_test)
test_accs.append(sum_loss / self.n_test)
if sum_loss < self.best_loss:
self.best_loss = sum_loss
self.best_epoch = epoch
serializers.save_hdf5('mlp.model', self.model)
print("update best loss")
#早期終了？
if self.xp.mean(self.save_accuracy) < sum_loss:
print("early stopping done")
break
#早期終了用配列にsum_accuracyを追加
self.save_accuracy = self.save_accuracy[1:]
append = self.xp.array([float(sum_loss)])
self.save_accuracy = self.xp.hstack((self.save_accuracy, append))
print("best_epoch: %d" % (self.best_epoch))
serializers.load_hdf5("mlp.model", self.model)
return train_accs, test_accs
class QNet:
# Hyper-Parameters
gamma = 0.95 # Discount factor
timestep_per_episode = 5000
initial_exploration = timestep_per_episode * 1 # Initial exploratoin. original: 5x10^4
replay_size = 32 # Replay (batch) size
hist_size = 2 # original: 4
data_index = 0
data_flag = False
loss_log = '../playground/Assets/log/'
def __init__(self, use_gpu, enable_controller, cnn_input_dim, feature_dim,
agent_count, other_input_dim, model):
self.use_gpu = use_gpu
self.num_of_actions = len(enable_controller)
self.enable_controller = enable_controller
self.cnn_input_dim = cnn_input_dim
self.feature_dim = feature_dim
self.agent_count = agent_count
self.other_input_dim = other_input_dim
self.data_size = self.timestep_per_episode
self.loss_log_file = self.loss_log + "loss.log"
self.loss_per_episode = 0
self.time_of_episode = 0
print("Initializing Q-Network...")
if model == 'None':
self.model = Chain(
conv1=L.Convolution2D(3 * self.hist_size, 32, 4, stride=2),
bn1=L.BatchNormalization(32),
conv2=L.Convolution2D(32, 32, 4, stride=2),
bn2=L.BatchNormalization(32),
conv3=L.Convolution2D(32, 32, 4, stride=2),
bn3=L.BatchNormalization(32),
# conv4=L.Convolution2D(64, 64, 4, stride=2),
# bn4=L.BatchNormalization(64),
l1=L.Linear(
self.feature_dim + self.other_input_dim * self.hist_size,
128),
l2=L.Linear(128, 128),
l3=L.Linear(128, 96),
l4=L.Linear(96, 64),
q_value=L.Linear(64, self.num_of_actions))
else:
with open(model, 'rb') as i:
self.model = pickle.load(i)
self.data_size = 0
if self.use_gpu >= 0:
self.model.to_gpu()
self.optimizer = optimizers.RMSpropGraves()
self.optimizer.setup(self.model)
# History Data : D=[s, a, r, s_dash, end_episode_flag]
self.d = [
np.zeros((self.agent_count, self.data_size, self.hist_size, 128,
128, 3),
dtype=np.uint8),
np.zeros((self.agent_count, self.data_size, self.hist_size,
self.other_input_dim),
dtype=np.uint8),
np.zeros((self.agent_count, self.data_size), dtype=np.uint8),
np.zeros((self.agent_count, self.data_size, 1), dtype=np.float32),
np.zeros((self.agent_count, self.data_size, 1), dtype=np.bool)
]
def _reshape_for_cnn(self, state, batch_size, hist_size, x, y):
state_ = np.zeros((batch_size, 3 * hist_size, 128, 128),
dtype=np.float32)
for i in range(batch_size):
if self.hist_size == 1:
state_[i] = state[i][0].transpose(2, 0, 1)
elif self.hist_size == 2:
state_[i] = np.c_[state[i][0], state[i][1]].transpose(2, 0, 1)
elif self.hist_size == 4:
state_[i] = np.c_[state[i][0], state[i][1], state[i][2],
state[i][3]].transpose(2, 0, 1)
return state_
def forward(self, state_cnn, state_other, action, reward, state_cnn_dash,
state_other_dash, episode_end):
num_of_batch = state_cnn.shape[0]
s_cnn = Variable(state_cnn)
s_oth = Variable(state_other)
s_cnn_dash = Variable(state_cnn_dash)
s_oth_dash = Variable(state_other_dash)
q = self.q_func(s_cnn, s_oth) # Get Q-value
max_q_dash_ = self.q_func(s_cnn_dash, s_oth_dash)
if self.use_gpu >= 0:
tmp = list(map(np.max, max_q_dash_.data.get()))
else:
tmp = list(map(np.max, max_q_dash_.data))
max_q_dash = np.asanyarray(tmp, dtype=np.float32)
if self.use_gpu >= 0:
target = np.array(q.data.get(), dtype=np.float32)
else:
target = np.array(q.data, dtype=np.float32)
for i in range(num_of_batch):
tmp_ = reward[i] + (1 -
episode_end[i]) * self.gamma * max_q_dash[i]
action_index = self.action_to_index(action[i])
target[i, action_index] = tmp_
if self.use_gpu >= 0:
loss = F.mean_squared_error(Variable(cuda.to_gpu(target)), q)
else:
loss = F.mean_squared_error(Variable(target), q)
return loss, q
def stock_experience(self, time, state_cnn, state_other, action, reward,
state_cnn_dash, state_other_dash, episode_end_flag):
for i in range(self.agent_count):
self.d[0][i][self.data_index] = state_cnn[i].copy()
self.d[1][i][self.data_index] = state_other[i].copy()
self.d[2][i][self.data_index] = action[i].copy()
self.d[3][i][self.data_index] = reward[i].copy()
self.d[4][i][self.data_index] = episode_end_flag
self.data_index += 1
if self.data_index >= self.data_size:
self.data_index -= self.data_size
self.data_flag = True
def experience_replay(self, time):
if self.initial_exploration < time:
# Pick up replay_size number of samples from the Data
replayRobotIndex = np.random.randint(0, self.agent_count,
self.replay_size)
if not self.data_flag: # during the first sweep of the History Data
replay_index = np.random.randint(0, self.data_index,
self.replay_size)
else:
replay_index = np.random.randint(0, self.data_size,
self.replay_size)
s_cnn_replay = np.ndarray(shape=(self.replay_size, self.hist_size,
128, 128, 3),
dtype=np.float32)
s_oth_replay = np.ndarray(shape=(self.replay_size, self.hist_size,
self.other_input_dim),
dtype=np.float32)
a_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.uint8)
r_replay = np.ndarray(shape=(self.replay_size, 1),
dtype=np.float32)
s_cnn_dash_replay = np.ndarray(shape=(self.replay_size,
self.hist_size, 128, 128, 3),
dtype=np.float32)
s_oth_dash_replay = np.ndarray(shape=(self.replay_size,
self.hist_size,
self.other_input_dim),
dtype=np.float32)
episode_end_replay = np.ndarray(shape=(self.replay_size, 1),
dtype=np.bool)
for i in range(self.replay_size):
s_cnn_replay[i] = np.asarray(
(self.d[0][replayRobotIndex[i]][replay_index[i]]),
dtype=np.float32)
s_oth_replay[i] = np.asarray(
(self.d[1][replayRobotIndex[i]][replay_index[i]]),
dtype=np.float32)
a_replay[i] = self.d[2][replayRobotIndex[i]][replay_index[i]]
r_replay[i] = self.d[3][replayRobotIndex[i]][replay_index[i]]
if (replay_index[i] + 1 >= self.data_size):
s_cnn_dash_replay[i] = np.array(
(self.d[0][replayRobotIndex[i]][replay_index[i] + 1 -
self.data_size]),
dtype=np.float32)
s_oth_dash_replay[i] = np.array(
(self.d[1][replayRobotIndex[i]][replay_index[i] + 1 -
self.data_size]),
dtype=np.float32)
else:
s_cnn_dash_replay[i] = np.array(
(self.d[0][replayRobotIndex[i]][replay_index[i] + 1]),
dtype=np.float32)
s_oth_dash_replay[i] = np.array(
(self.d[1][replayRobotIndex[i]][replay_index[i] + 1]),
dtype=np.float32)
episode_end_replay[i] = self.d[4][replayRobotIndex[i]][
replay_index[i]]
s_cnn_replay = self._reshape_for_cnn(s_cnn_replay,
self.replay_size,
self.hist_size, 128, 128)
s_cnn_dash_replay = self._reshape_for_cnn(s_cnn_dash_replay,
self.replay_size,
self.hist_size, 128, 128)
s_cnn_replay /= 255.0
s_oth_replay /= 255.0
s_cnn_dash_replay /= 255.0
s_oth_dash_replay /= 255.0
if self.use_gpu >= 0:
s_cnn_replay = cuda.to_gpu(s_cnn_replay)
s_oth_replay = cuda.to_gpu(s_oth_replay)
s_cnn_dash_replay = cuda.to_gpu(s_cnn_dash_replay)
s_oth_dash_replay = cuda.to_gpu(s_oth_dash_replay)
# Gradient-based update
loss, _ = self.forward(s_cnn_replay, s_oth_replay, a_replay,
r_replay, s_cnn_dash_replay,
s_oth_dash_replay, episode_end_replay)
send_loss = loss.data
with open(self.loss_log_file, 'a') as the_file:
the_file.write(str(time) + "," + str(send_loss) + "\n")
self.loss_per_episode += loss.data
self.time_of_episode += 1
self.model.zerograds()
loss.backward()
self.optimizer.update()
def q_func(self, state_cnn, state_other):
if self.use_gpu >= 0:
num_of_batch = state_cnn.data.get().shape[0]
else:
num_of_batch = state_cnn.data.shape[0]
h1 = F.tanh(self.model.bn1(self.model.conv1(state_cnn)))
h2 = F.tanh(self.model.bn2(self.model.conv2(h1)))
h3 = F.tanh(self.model.bn3(self.model.conv3(h2)))
# h4 = F.tanh(self.model.bn4(self.model.conv4(h3)))
# h5 = F.tanh(self.model.bn5(self.model.conv5(h4)))
h4_ = F.concat(
(F.reshape(h3, (num_of_batch, self.feature_dim)),
F.reshape(state_other,
(num_of_batch, self.other_input_dim * self.hist_size))),
axis=1)
h6 = F.relu(self.model.l1(h4_))
h7 = F.relu(self.model.l2(h6))
h8 = F.relu(self.model.l3(h7))
h9 = F.relu(self.model.l4(h8))
q = self.model.q_value(h9)
return q
def e_greedy(self, state_cnn, state_other, epsilon, reward):
s_cnn = Variable(state_cnn)
s_oth = Variable(state_other)
q = self.q_func(s_cnn, s_oth)
q = q.data
if self.use_gpu >= 0:
q_ = q.get()
else:
q_ = q
index_action = np.zeros((self.agent_count), dtype=np.uint8)
print(("agent"), end=' ')
for i in range(self.agent_count):
if np.random.rand() < epsilon:
index_action[i] = np.random.randint(0, self.num_of_actions)
print(("[%02d] Random(%2d)reward(%06.2f)" %
(i, index_action[i], reward[i])),
end=' ')
else:
index_action[i] = np.argmax(q_[i])
print(("[%02d]!Greedy(%2d)reward(%06.2f)" %
(i, index_action[i], reward[i])),
end=' ')
if i % 5 == 4:
print(("\n "), end=' ')
del q_
return self.index_to_action(index_action), q
def index_to_action(self, index_of_action):
index = np.zeros((self.agent_count), dtype=np.uint8)
for i in range(self.agent_count):
index[i] = self.enable_controller[index_of_action[i]]
return index
def action_to_index(self, action):
return self.enable_controller.index(action)