def train(opt):
#print('decay', opt.num_decay_epochs)
if torch.cuda.is_available():
torch.cuda.manual_seed(123)
else:
torch.manual_seed(123)
if os.path.isdir(opt.log_path):
shutil.rmtree(opt.log_path)
os.makedirs(opt.log_path)
writer = SummaryWriter(opt.log_path)
env = Tetris(width=opt.width, height=opt.height,
block_size=opt.block_size) #高さ、幅、1ブロックの大きさを指定
model = DeepQNetwork() #インスタンス生成
#optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
optimizer = torch.optim.SGD(model.parameters(), lr=opt.lr)
criterion = nn.MSELoss()
state = env.reset() # 初期状態 tensor([0., 0., 0., 0.])
if torch.cuda.is_available():
model.cuda()
state = state.cuda()
replay_memory = deque(maxlen=opt.replay_memory_size) #maxで30000、
epoch = 0
while epoch < opt.num_epochs: # 指定したエポック数繰り返す
#1ピース目の取りうる全ての行動に対して、それぞれ状態を計算 {(左から何番目か,何回転か):tensor([,,,]),*n}
next_steps = env.get_next_states()
# εグリーディー的なやつ
#epsilon = opt.final_epsilon + (max(opt.num_decay_epochs - epoch, 0) * ( #num_decay_epochs以降一定
# opt.initial_epsilon - opt.final_epsilon) / opt.num_decay_epochs)
epsilon = opt.initial_epsilon - opt.initial_epsilon * epoch / opt.num_epochs #直線
u = random() # 0～1
random_action = u <= epsilon # True, False
next_actions, next_states = zip(
*next_steps.items()) #next_stepsのkeyとvalueを取得 #( , )*n
next_states = torch.stack(next_states) # tensor([[ , , , ],*n])
if torch.cuda.is_available():
next_states = next_states.cuda()
model.eval()
with torch.no_grad():
predictions = model(
next_states
)[:, 0] #DeepQNetworkのforward #tensor([,～,])これはそれぞれの行動に対するQ値のようなもの
model.train()
# next_stepsのインデックスをランダムor最適で指定
if random_action: # ランダムな行動
index = randint(0, len(next_steps) - 1)
else: # 最適な行動(最大のpredictionsに基づく)
index = torch.argmax(predictions).item()
# 行動と次の状態を決定
next_state = next_states[
index, :] #ある行動を選択したときの次の状態 #tensor([ , , , ])
action = next_actions[index] #行動 #(左から何番目か,何回転か)
reward, done = env.step(
action, epoch, render=False) #行動を実行、報酬(スコア)を求める、溢れた場合done=True、描画
if torch.cuda.is_available():
next_state = next_state.cuda()
replay_memory.append(
[state, reward, next_state, done]
) #deque([[tensor([0., 0., 0., 0.]), 1, tensor([0., 0., 2., 4.]), False]],..., maxlen=30000)
if done: # 溢れた場合 or 上限100手
final_score = env.score
final_tetrominoes = env.tetrominoes
final_cleared_lines = env.cleared_lines
cleared_lines1 = env.cleared_lines1
cleared_lines2 = env.cleared_lines2
cleared_lines3 = env.cleared_lines3
cleared_lines4 = env.cleared_lines4
state = env.reset() # 初期状態 tensor([0., 0., 0., 0.])
if torch.cuda.is_available():
state = state.cuda()
else: # 溢れてない場合
state = next_state # 状態を更新 tensor([0., 1., 2., 5.])とか
continue #while epoch～に戻る
#if len(replay_memory) < opt.replay_memory_size / 1000: #溢れた場合判定(累計ピースが3000以下ならcontinue)
#continue #pass
# 累計ピースが3000に到達した後、溢れる毎に以下を実行
epoch += 1
batch = sample(
replay_memory, min(len(replay_memory), opt.batch_size)
) #replay_memoryからbatch_size個ランダムに取り出す(len(replay_memory) < opt.batch_sizeのときはlen(replay_memory)個取り出す)
replay_memory.clear() #中身を全消去
state_batch, reward_batch, next_state_batch, done_batch = zip(*batch)
state_batch = torch.stack(
tuple(state for state in
state_batch)) #tensor([[0., 26., 16., 62.],*batch_size個])
reward_batch = torch.from_numpy(
np.array(reward_batch,
dtype=np.float32)[:, None]) #tensor([[1.],*batch_size個])
next_state_batch = torch.stack(
tuple(
state for state in
next_state_batch)) #tensor([[0., 32., 13., 72.],*batch_size個])
if torch.cuda.is_available():
state_batch = state_batch.cuda()
reward_batch = reward_batch.cuda()
next_state_batch = next_state_batch.cuda()
q_values = model(
state_batch) #予測Q値、q_values=tensor([[0.1810],*batch_size個])
model.eval()
with torch.no_grad():
next_prediction_batch = model(next_state_batch) #次の状態に対する予測Q値
model.train()
# Q値の正解値を更新式で求める
y_batch = torch.cat(
tuple(reward if done else reward + opt.gamma * prediction
for reward, done, prediction in zip(
reward_batch, done_batch, next_prediction_batch)))[:,
None]
optimizer.zero_grad() #最適化アルゴリズム
loss = criterion(q_values, y_batch) #損失関数はmse、q_values:予測値、y_batch:正解値
"""
length = len(q_values)
errors = np.zeros([length])
print('size', len(q_values), len(y_batch))
for i in range(length):
print('Q', q_values[i])
print('Y', y_batch[i])
errors[i] = (q_values[i] - y_batch[i]) ** 2
error = np.mean(errors)
print('error', error)
print('loss',loss)
"""
loss.backward()
optimizer.step()
if epoch % 10 == 0:
print(
"Epoch: {}/{}, Action: {}, Score: {}, Tetrominoes {}, Cleared lines: {}"
.format(epoch, opt.num_epochs, action, final_score,
final_tetrominoes, final_cleared_lines))
#学習中のスコアをcsvに記録
if epoch == 1:
with open('Score_train.csv', mode='w',
newline="") as Score_train_Record:
writer = csv.writer(Score_train_Record)
writer.writerow([
epoch, final_tetrominoes, final_score, final_cleared_lines,
cleared_lines1, cleared_lines2, cleared_lines3,
cleared_lines4
])
else:
with open('Score_train.csv', mode='a',
newline="") as Score_train_Record:
writer = csv.writer(Score_train_Record)
writer.writerow([
epoch, final_tetrominoes, final_score, final_cleared_lines,
cleared_lines1, cleared_lines2, cleared_lines3,
cleared_lines4
])
"""
writer.add_scalar('Train/Score', final_score, epoch - 1)
writer.add_scalar('Train/Tetrominoes', final_tetrominoes, epoch - 1)
writer.add_scalar('Train/Cleared lines', final_cleared_lines, epoch - 1)
"""
if epoch > 0 and epoch % opt.save_interval == 0:
torch.save(model, "{}/tetris2_{}".format(
opt.saved_path, epoch)) #定期的にモデルをtrained_modelsに保存
if final_tetrominoes > 500: #ミノ数が500を超えたモデルの重みとバイアスをcsvに保存
save_model_parameter(model)
torch.save(model,
"{}/tetris2".format(opt.saved_path)) #学習後のモデルをtrained_modelsに保存
def train(opt):
if torch.cuda.is_available():
torch.cuda.manual_seed(123)
else:
torch.manual_seed(123)
# TensorBoard
if os.path.isdir(opt.log_path):
shutil.rmtree(opt.log_path)
os.makedirs(opt.log_path)
writer = SummaryWriter(opt.log_path)
# Modelo
CHECKPOINT_FILE = opt.saved_path + "/" + opt.checkpoint_name
if opt.load:
if os.path.isfile(CHECKPOINT_FILE):
print("--> Carregando Checkpoint '{}'.".format(CHECKPOINT_FILE))
if torch.cuda.is_available():
model = torch.load(CHECKPOINT_FILE)
else:
model = torch.load(CHECKPOINT_FILE, map_location=lambda storage, loc: storage)
print("--> Checkpoint Carregado '{}'.".format(CHECKPOINT_FILE))
else:
print("--> Checkpoint '{}' não encontrado.".format(CHECKPOINT_FILE))
model = DeepQNetwork()
else:
model = DeepQNetwork()
optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
criterion = nn.MSELoss()
# Environment
env = Tetris(width=opt.width, height=opt.height)
state = env.reset()
if torch.cuda.is_available():
model.cuda()
state = state.cuda()
replay_memory = deque(maxlen=opt.replay_memory_size)
epoch = 0
prev_loss = 0
# Épocas do Checkpoint
if opt.load and "_" in opt.checkpoint_name:
start_epoch = opt.checkpoint_name.split("_")[-1]
epoch = int(start_epoch)
print("Checkpoint com {} épocas.".format(epoch))
# Loop de Treino
while epoch < opt.num_epochs:
next_steps = env.get_next_states()
# Exploração ou Explotação
epsilon = opt.final_epsilon + (max(opt.num_decay_epochs - epoch, 0) * (
opt.initial_epsilon - opt.final_epsilon) / opt.num_decay_epochs)
u = random()
random_action = u <= epsilon
next_actions, next_states = zip(*next_steps.items())
next_states = torch.stack(next_states)
if torch.cuda.is_available():
next_states = next_states.cuda()
model.eval()
with torch.no_grad():
predictions = model(next_states)[:, 0]
model.train()
if random_action:
index = randint(0, len(next_steps) - 1)
else:
index = torch.argmax(predictions).item()
next_state = next_states[index, :]
action = next_actions[index]
reward, done = env.step(action, render=True)
if torch.cuda.is_available():
next_state = next_state.cuda()
replay_memory.append([state, reward, next_state, done])
if done:
final_score = env.score
final_tetrominoes = env.tetrominoes
final_cleared_lines = env.cleared_lines
state = env.reset()
if torch.cuda.is_available():
state = state.cuda()
else:
state = next_state
continue
# Replay Buffer
if len(replay_memory) < opt.replay_memory_size / 10:
print("replay_memory ", len(replay_memory))
continue
epoch += 1
batch = sample(replay_memory, min(len(replay_memory), opt.batch_size))
state_batch, reward_batch, next_state_batch, done_batch = zip(*batch)
state_batch = torch.stack(tuple(state for state in state_batch))
reward_batch = torch.from_numpy(np.array(reward_batch, dtype=np.float32)[:, None])
next_state_batch = torch.stack(tuple(state for state in next_state_batch))
# Aprendizado
if torch.cuda.is_available():
state_batch = state_batch.cuda()
reward_batch = reward_batch.cuda()
next_state_batch = next_state_batch.cuda()
q_values = model(state_batch)
model.eval()
with torch.no_grad():
next_prediction_batch = model(next_state_batch)
model.train()
y_batch = torch.cat(
tuple(reward if done else reward + opt.gamma * prediction for reward, done, prediction in
zip(reward_batch, done_batch, next_prediction_batch)))[:, None]
optimizer.zero_grad()
loss = criterion(q_values, y_batch)
loss.backward()
optimizer.step()
prev_loss = loss.item()
print("Epoch: {}/{}, Action: {}, Score: {}, Tetrominoes {}, Cleared lines: {}".format(
epoch,
opt.num_epochs,
action,
final_score,
final_tetrominoes,
final_cleared_lines))
writer.add_scalar('Train/Score', final_score, epoch - 1)
writer.add_scalar('Train/Tetrominoes', final_tetrominoes, epoch - 1)
writer.add_scalar('Train/Cleared lines', final_cleared_lines, epoch - 1)
if epoch > 0 and epoch % opt.save_interval == 0:
torch.save(model, "{}/{}_{}".format(opt.saved_path, opt.saved_name, epoch))
torch.save(model, "{}/{}".format(opt.saved_path, opt.saved_name))
def train(opt):
if torch.cuda.is_available():
# 随机数种子seed确定时，模型的训练结果将始终保持一致
torch.cuda.manual_seed(123)
else:
torch.manual_seed(123)
if os.path.isdir(opt.log_path):
shutil.rmtree(opt.log_path)
os.makedirs(opt.log_path)
writer = SummaryWriter(opt.log_path)
env = Tetris(width=opt.width, height=opt.height, block_size=opt.block_size)
model = DeepQNetwork()
optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
criterion = nn.MSELoss()
state = env.reset()
if torch.cuda.is_available():
model.cuda()
state = state.cuda()
replay_memory = deque(maxlen=opt.replay_memory_size)
epoch = 0
while epoch < opt.num_epochs:
# 得到所有可能的下落方块
next_steps = env.get_next_states()
# Exploration or exploitation
epsilon = opt.final_epsilon + (
max(opt.num_decay_epochs - epoch, 0) *
(opt.initial_epsilon - opt.final_epsilon) / opt.num_decay_epochs)
u = random()
random_action = u <= epsilon
# 下一步落下的横向坐标以及旋转，以及得到下方方块的board状态
next_actions, next_states = zip(*next_steps.items())
next_states = torch.stack(next_states)
if torch.cuda.is_available():
next_states = next_states.cuda()
model.eval()
with torch.no_grad():
predictions = model(next_states)[:, 0]
model.train()
# 采取的动作
if random_action:
index = randint(0, len(next_steps) - 1)
else:
index = torch.argmax(predictions).item()
next_state = next_states[index, :]
action = next_actions[index]
reward, done = env.step(action, render=False)
if torch.cuda.is_available():
next_state = next_state.cuda()
replay_memory.append([state, reward, next_state, done])
if done:
final_score = env.score
final_tetrominoes = env.tetrominoes
final_cleared_lines = env.cleared_lines
state = env.reset()
if torch.cuda.is_available():
state = state.cuda()
else:
state = next_state
continue
if len(replay_memory) < opt.replay_memory_size / 10:
continue
epoch += 1
batch = sample(replay_memory, min(len(replay_memory), opt.batch_size))
'''
a = [2, 3, 4], b = [5, 6, 7], c = [a, b]
e, f, g = zip(*c)
e = (2, 5), f = (3, 6), g = (4, 7) 类型为tuple
'''
state_batch, reward_batch, next_state_batch, done_batch = zip(*batch)
state_batch = torch.stack(tuple(state for state in state_batch))
reward_batch = torch.from_numpy(
np.array(reward_batch, dtype=np.float32)[:, None])
next_state_batch = torch.stack(
tuple(state for state in next_state_batch))
if torch.cuda.is_available():
state_batch = state_batch.cuda()
reward_batch = reward_batch.cuda()
next_state_batch = next_state_batch.cuda()
q_values = model(state_batch)
model.eval()
# Q_target
with torch.no_grad():
next_prediction_batch = model(next_state_batch)
model.train()
y_batch = torch.cat(
tuple(reward if done else reward + opt.gamma * prediction
for reward, done, prediction in zip(
reward_batch, done_batch, next_prediction_batch)))[:,
None]
optimizer.zero_grad()
loss = criterion(q_values, y_batch)
loss.backward()
optimizer.step()
print(
"Epoch: {}/{}, Action: {}, Score: {}, Tetrominoes {}, Cleared lines: {}"
.format(epoch, opt.num_epochs, action, final_score,
final_tetrominoes, final_cleared_lines))
writer.add_scalar('Train/Score', final_score, epoch - 1)
writer.add_scalar('Train/Tetrominoes', final_tetrominoes, epoch - 1)
writer.add_scalar('Train/Cleared lines', final_cleared_lines,
epoch - 1)
if epoch > 0 and epoch % opt.save_interval == 0:
torch.save(model, "{}/tetris_{}".format(opt.saved_path, epoch))
torch.save(model, "{}/tetris".format(opt.saved_path))
def train(opt):
if torch.cuda.is_available():
torch.cuda.manual_seed(123)
else:
torch.manual_seed(123)
if os.path.isdir(opt.log_path):
shutil.rmtree(opt.log_path)
os.makedirs(opt.log_path)
writer = SummaryWriter(opt.log_path)
env = Tetris(width=opt.width, height=opt.height, block_size=opt.block_size)
model = DeepQNetwork()
model_target = DeepQNetwork()
optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
criterion = nn.MSELoss()
state = env.reset()
if torch.cuda.is_available():
model.cuda()
model_target.cuda()
state = state.cuda()
if opt.PER:
replay_memory = Memory(capacity=opt.replay_memory_size)
else:
replay_memory = deque(maxlen=opt.replay_memory_size)
epoch = 0
warmup_epoch = 0
while epoch < opt.num_epochs:
next_steps = env.get_next_states()
# Exploration or exploitation
epsilon = opt.final_epsilon + (
max(opt.num_decay_epochs - epoch, 0) *
(opt.initial_epsilon - opt.final_epsilon) / opt.num_decay_epochs)
u = random()
random_action = u <= epsilon
next_actions, next_states = zip(*next_steps.items())
next_states = torch.stack(next_states)
if torch.cuda.is_available():
next_states = next_states.cuda()
model.eval()
with torch.no_grad():
predictions = model(next_states)[:, 0]
model.train()
if random_action:
index = randint(0, len(next_steps) - 1)
else:
index = torch.argmax(predictions).item()
next_state = next_states[index, :]
action = next_actions[index]
reward, done = env.step(action, render=True)
if torch.cuda.is_available():
next_state = next_state.cuda()
if opt.PER:
experience = state, action, reward, next_state, done
replay_memory.store(experience)
else:
replay_memory.append([state, reward, next_state, done])
if done:
final_score = env.score
final_tetrominoes = env.tetrominoes
final_cleared_lines = env.cleared_lines
state = env.reset()
if torch.cuda.is_available():
state = state.cuda()
else:
state = next_state
continue
warmup_epoch += 1
if warmup_epoch < opt.learning_starts:
continue
epoch += 1
if opt.PER:
tree_idx, batch = replay_memory.sample(opt.batch_size)
else:
batch = sample(replay_memory,
min(len(replay_memory), opt.batch_size))
state_batch, _, reward_batch, next_state_batch, done_batch = zip(
*batch)
state_batch = torch.stack(tuple(state for state in state_batch))
reward_batch = torch.from_numpy(
np.array(reward_batch, dtype=np.float32)[:, None])
next_state_batch = torch.stack(
tuple(state for state in next_state_batch))
if torch.cuda.is_available():
state_batch = state_batch.cuda()
reward_batch = reward_batch.cuda()
next_state_batch = next_state_batch.cuda()
q_values = model(state_batch)
model_target.eval()
with torch.no_grad():
next_prediction_batch = model_target(next_state_batch)
model_target.train()
y_batch = torch.cat(
tuple(reward if done else reward + opt.gamma * prediction
for reward, done, prediction in zip(
reward_batch, done_batch, next_prediction_batch)))[:,
None]
optimizer.zero_grad()
loss = criterion(q_values, y_batch)
loss.backward()
optimizer.step()
model.eval()
model_target.eval()
if opt.PER:
with torch.no_grad():
if torch.cuda.is_available():
replay_memory.batch_update(
tree_idx,
np.abs(q_values.detach().cpu().numpy() -
y_batch.cpu().numpy()))
else:
replay_memory.batch_update(
tree_idx,
np.abs(q_values.detach().numpy() - y_batch.numpy()))
# Update target model <- model
if epoch % opt.target_update_freq == 0:
with torch.no_grad():
model_target.load_state_dict(model.state_dict())
model_target.train()
model.eval()
print(
"Epoch: {}/{}, Action: {}, Score: {}, Tetrominoes {}, Cleared lines: {}"
.format(epoch, opt.num_epochs, action, final_score,
final_tetrominoes, final_cleared_lines))
writer.add_scalar('Train/Score', final_score, epoch - 1)
writer.add_scalar('Train/Tetrominoes', final_tetrominoes, epoch - 1)
writer.add_scalar('Train/Cleared lines', final_cleared_lines,
epoch - 1)
if (epoch > 0
and epoch % opt.save_interval) == 0 or final_score >= 10000.0:
torch.save(model, "{}/tetris_{}".format(opt.saved_path, epoch))
torch.save(model, "{}/tetris".format(opt.saved_path))
