def train(env, config):
"""
"""
t0 = time.time()
save_models_path = str(config["locexp"])
memory = ReplayBuffer((8, ), (1, ), config["buffer_size"], config["seed"],
config["device"])
memory.load_memory(config["buffer_path"])
agent = Agent(state_size=8, action_size=4, config=config)
if config["idx"] < memory.idx:
memory.idx = config["idx"]
print("memory idx ", memory.idx)
for t in range(config["predicter_time_steps"]):
text = "Train Predicter {} \ {} time {} \r".format(
t, config["predicter_time_steps"], time_format(time.time() - t0))
print(text, end='')
agent.learn(memory)
if t % int(config["eval"]) == 0:
print(text)
agent.save(save_models_path + "/models/{}-".format(t))
#agent.test_predicter(memory)
agent.test_q_value(memory)
agent.eval_policy()
agent.eval_policy(True, 1)
def train(env, config):
"""
"""
t0 = time.time()
memory = ReplayBuffer((8,), (1,), config["expert_buffer_size"], config["device"])
memory.load_memory(config["buffer_path"])
agent = Agent(state_size=8, action_size=4, config=config)
memory.idx = config["idx"]
#for i in range(10):
# print("state", memory.obses[i])
# sys.exit()
print("memroy idx ",memory.idx)
if config["mode"] == "predict":
for t in range(config["predicter_time_steps"]):
text = "Train Predicter {} \ {} time {} \r".format(t, config["predicter_time_steps"], time_format(time.time() - t0))
print(text, end = '')
agent.learn_predicter(memory)
if t % 2000 == 0:
# agent.test_predicter(memory)
agent.save("pytorch_models-{trained_predicter}/")
return
if config["mode"] == "iql":
agent.test_predicter(memory)
for t in range(config["predicter_time_steps"]):
text = "Train Predicter {} \ {} time {} \r".format(t, config["predicter_time_steps"], time_format(time.time() - t0))
print(text, end = '')
agent.learn(memory)
if t % 100 == 0:
print(text)
agent.test_predicter(memory)
agent.test_q_value(memory)
# agent.test_policy()
if config["mode"] == "dqn":
print("mode dqn")
agent.dqn_train()
return
from replay_buffer import ReplayBuffer
import cv2
memory = ReplayBuffer((64, 64, 3), (1, ), 50000, "cuda")
memory.load_memory("pacman_expert_memory-9000")
state = memory.obses[0]
next_state = memory.next_obses[0]
print(state.shape)
cv2.imshow('HelloWorld', state)
cv2.waitKey(0)
cv2.imshow('HelloWorld', next_state)
cv2.waitKey(0)
def train_agent(env, config):
"""
Args:
"""
# create CNN convert the [1,3,84,84] to [1, 200]
now = datetime.now()
dt_string = now.strftime("%d_%m_%Y_%H:%M:%S")
torch.manual_seed(config["seed"])
np.random.seed(config["seed"])
#pathname = str(args.locexp) + "/" + str(args.env_name) + '_agent_' + str(args.policy)
#pathname += "_batch_size_" + str(args.batch_size) + "_lr_act_" + str(args.lr_actor)
#pathname += "_lr_critc_" + str(args.lr_critic) + "_lr_decoder_"
pathname = dt_string
tensorboard_name = str(config["locexp"]) + '/runs/' + pathname
agent = DQNAgent(state_size=200,
action_size=env.action_space.n,
config=config)
writer = SummaryWriter(tensorboard_name)
print("action_size {}".format(env.action_space.n))
# eval_policy(env, agent, writer, 0, config)
memory = ReplayBuffer((3, config["size"], config["size"]),
(1, ), config["expert_buffer_size"],
int(config["image_pad"]), config["device"])
if config["create_buffer"]:
create_buffer(env, memory, config)
memory.load_memory("/export/leiningc/" + config["buffer_path"])
else:
print("load Buffer")
memory.load_memory("/export/leiningc/" + config["buffer_path"])
print("Buffer size {}".format(memory.idx))
eps = config["eps_start"]
eps_end = config["eps_end"]
eps_decay = config["eps_decay"]
scores_window = deque(maxlen=100)
scores = []
t0 = time.time()
for i_episode in range(config["train_episodes"]):
obs = env.reset()
score = 0
for t in range(config["max_t"]):
action = agent.act(obs, eps)
# action = env.action_space.sample()
next_obs, reward, done_no_max, _ = env.step(action)
done = done_no_max
if t + 1 == config["max_t"]:
print("t ", t)
done = 0
memory.add(obs, action, reward, next_obs, done, done_no_max)
agent.step(memory, writer)
obs = next_obs
eps = max(eps_end, eps_decay * eps) # decrease epsilon
score += reward
if done:
break
scores_window.append(score) # save most recent scor
scores.append(score) # save most recent score
ave_score = np.mean(scores_window)
writer.add_scalar("ave_score", ave_score, i_episode)
writer.add_scalar("episode_score", score, i_episode)
print(
'\rEpisode {} score {} \tAverage Score: {:.2f} eps: {:.2f} time: {}'
.format(i_episode, score, np.mean(scores_window), eps,
time_format(time.time() - t0)),
end="")
if i_episode % config["eval"] == 0:
eval_policy(env, agent, writer, i_episode, config)
agent.save(
str(config["locexp"]) + "/models/eval-{}/".format(i_episode))
print(
'Episode {} Average Score: {:.2f} eps: {:.2f} time: {}'.
format(i_episode, np.mean(scores_window), eps,
time_format(time.time() - t0)), )
