def __init__(self, state_dim, action_dim, learning_rate=0.001, reward_decay=0.99, e_greedy=0.9): self.action_dim = action_dim self.state_dim = state_dim self.lr = learning_rate self.gamma = reward_decay # in according to the parameters in the formulation. self.epsilon = e_greedy self.EPS_START = 0.9 self.EPS_END = 0.05 self.EPS_DECAY = 30000 # this decay is to slow. # TO DO: figure out the relationship between the decay and the totoal step. # try to use a good strategy to solve this problem. use_cuda = torch.cuda.is_available() self.LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor self.FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor self.model = QNet(self.state_dim, self.action_dim).cuda() if use_cuda else QNet( self.state_dim, self.action_dim) self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr) # self.scheduler = optim.StepLR(self.optimizer, step_size=10000, gamma=0.5) # the learning rate decrease by a factor gamma every 10000 step_size. util.weights_init(self.model)
def __init__(self, env): super(DDPG, self).__init__() pi_net = PiNet(self.ns, self.na) self.pi_net = pi_net.to(self.device) pi_target = PiNet(self.ns, self.na) self.pi_target = pi_target.to(self.device) self.load_state_dict(self.pi_target, self.pi_net.state_dict()) q_net = QNet(self.ns, self.na) self.q_net = q_net.to(self.device) q_target = QNet(self.ns, self.na) self.q_target = q_target.to(self.device) self.load_state_dict(self.q_target, self.q_net.state_dict()) self.optimizer_q = torch.optim.Adam(self.q_net.parameters(), lr=self.lr_q, betas=(0.9, 0.999), weight_decay=1e-2) self.optimizer_p = torch.optim.Adam(self.pi_net.parameters(), lr=self.lr_p, betas=(0.9, 0.999), weight_decay=0) self.noise = OrnsteinUhlenbeckActionNoise( torch.zeros(1, self.na).to(self.device), self.epsilon * torch.ones(1, self.na).to(self.device))
def __init__(self, args, state_size, action_size, seed): self.state_size = state_size self.action_size = action_size self.seed = random.seed(seed) self.per = args.per self.dueling = args.dueling self.buffer_size = args.buffer_size self.batch_size = args.batch_size self.gamma = args.gamma self.tau = args.tau self.lr = args.learning_rate self.update_freq = args.update_every # Q-Network if self.dueling: self.local_qnet = DuelingQNet(state_size, action_size, seed).to(device) self.target_qnet = DuelingQNet(state_size, action_size, seed).to(device) else: self.local_qnet = QNet(state_size, action_size, seed).to(device) self.target_qnet = QNet(state_size, action_size, seed).to(device) self.optimizer = optim.Adam(self.local_qnet.parameters(), lr=self.lr) # Replay Memory if self.per: self.memory = PrioritizedReplayMemory(args, self.buffer_size) else: self.memory = ReplayMemory(action_size, self.buffer_size, self.batch_size, seed) self.t_step = 0 # init time step for updating every UPDATE_EVERY steps
def __init__(self, state_size, action_size, seed): self.state_size = state_size self.action_size = action_size self.seed = random.seed(seed) self.qnetwork_local = QNet(state_size, action_size, seed).to(device) self.qnetwork_target = QNet(state_size, action_size, seed).to(device) self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR) # Replay Memory self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed) self.t_step = 0
def __init__(self): #Creating environment self.env = gym.make(settings.env_name) self.env.seed(settings.seed) self.env.action_space.seed(settings.seed) self.state_space = self.env.observation_space.shape[0] self.action_space = self.env.action_space.shape[0] self.obs_normalizer = Normalizer(self.state_space) self.device = torch.device(settings.device) self.writer = SummaryWriter( 'runs/' + settings.env_name + "_" + settings.algo + '_{}_{}_{}'.format(p.alpha, p.ex_alpha, settings.seed)) #Initializing common networks and their optimizers self.exploitory_policy = GaussianPolicy( self.state_space, self.action_space).to(self.device) self.exploitory_Q = QNet(self.state_space, self.action_space).to(self.device) self.exploitory_Q_target = QNet(self.state_space, self.action_space).to(self.device) self.exploitory_policy_optim = Adam( self.exploitory_policy.parameters(), lr=p.lr) self.exploitory_Q_optim = Adam(self.exploitory_Q.parameters(), lr=p.lr) self.target_update(self.exploitory_Q_target, self.exploitory_Q, 1.0) p.alpha = torch.Tensor([p.alpha]).to(self.device) if settings.automatic_entropy_tuning: self.target_entropy = -torch.prod( torch.Tensor(self.env.action_space.shape).to( self.device)).item() self.log_alpha = torch.zeros(1, requires_grad=True, device=self.device) self.alpha_optim = Adam([self.log_alpha], lr=p.lr) if settings.automatic_ex_entropy_tuning: self.ex_target_entropy = -torch.prod( torch.Tensor(self.env.action_space.shape).to( self.device)).item() self.ex_log_alpha = torch.zeros(1, requires_grad=True, device=self.device) self.ex_alpha_optim = Adam([self.log_alpha], lr=p.lr) if settings.reward_model == 'novelty': self.ex_reward_model = Novelty(self.state_space, self.device)
def __init__(self, *largs, **kwargs): super(SSPG, self).__init__(*largs, **kwargs) pi_net = PiNet(self.ns, self.na, distribution='Normal') self.pi_net = pi_net.to(self.device) pi_target = PiNet(self.ns, self.na, distribution='Normal') self.pi_target = pi_target.to(self.device) self.load_state_dict(self.pi_target, self.pi_net.state_dict()) q_net_1 = QNet(self.ns, self.na) self.q_net_1 = q_net_1.to(self.device) q_target_1 = QNet(self.ns, self.na) self.q_target_1 = q_target_1.to(self.device) self.load_state_dict(self.q_target_1, self.q_net_1.state_dict()) q_net_2 = QNet(self.ns, self.na) self.q_net_2 = q_net_2.to(self.device) q_target_2 = QNet(self.ns, self.na) self.q_target_2 = q_target_2.to(self.device) self.load_state_dict(self.q_target_2, self.q_net_2.state_dict()) self.optimizer_q_1 = torch.optim.Adam(self.q_net_1.parameters(), lr=self.lr_q, betas=(0.9, 0.999), weight_decay=self.weight_decay_q) self.optimizer_q_2 = torch.optim.Adam(self.q_net_2.parameters(), lr=self.lr_q, betas=(0.9, 0.999), weight_decay=self.weight_decay_q) self.optimizer_p = torch.optim.Adam(self.pi_net.parameters(), lr=self.lr_p, betas=(0.9, 0.999), weight_decay=self.weight_decay_p) if self.entropy_tunning: self.target_entropy = -torch.prod( torch.Tensor(self.na).to(self.device)).item() self.log_alpha = torch.tensor([0.], requires_grad=True, device=self.device) self.optimizer_alpha = torch.optim.Adam([self.log_alpha], lr=self.lr_q) self.alpha = float(self.log_alpha.exp())
def test(L, mouse_initial_indices, rewardlist, actions_list): online_net = QNet(3, 4).to(device) online_net.load_state_dict( torch.load("./qlearning_model", map_location=device)) env = deepcopy(L) done = False eaubue = 0. steps = 0 score = 0 if mouse_initial_indices is None: all_possible_starting_positions = np.array([*np.where(L == 1)]).T mouse_initial_indices = all_possible_starting_positions[ np.random.choice(range(len(all_possible_starting_positions)))] state = np.array(mouse_initial_indices) state = torch.Tensor(state).to(device) state = state.unsqueeze(0) def progress_loop(done, steps, state, score, eaubue): steps += 1 action = get_action(state, online_net, 1, env, True, eaubue) displacement = np.array(actions_list[action]) newstate = state + torch.Tensor(displacement).to(device) if env[int(newstate[0][0].tolist()), int(newstate[0][1].tolist())] != 0: next_state = newstate displayer.main_canva.move(displayer.mouse, *(displacement * displayer.square_size)) reward = rewardlist[env[int(newstate[0][0].tolist()), int(newstate[0][1].tolist())]] if env[int(newstate[0][0].tolist()), int(newstate[0][1].tolist())] == 2: done = True if env[int(newstate[0][0].tolist()), int(newstate[0][1].tolist() )] == 4: #if the mouse is in the water env[int(newstate[0][0].tolist()), int(newstate[0][1].tolist())] = 5 #there is no more water eaubue = 1. else: next_state = state reward = rewardlist[0] score += reward state = next_state print('position : ', state.tolist()[0], score) if done is False: displayer.window.after( 800, lambda: progress_loop(done, steps, state, score, eaubue)) displayer = Displayer() displayer.create_labyrinth(L, mouse_initial_indices) progress_loop(done, steps, state, score, 0.) displayer.window.mainloop()
def main(): env = gym.make(env_name) env.seed(500) torch.manual_seed(500) num_inputs = env.observation_space.shape[0] num_actions = env.action_space.n print('state size:', num_inputs) print('action size:', num_actions) net = QNet(num_inputs, num_actions) optimizer = optim.Adam(net.parameters(), lr=lr) writer = SummaryWriter('logs') net.to(device) net.train() running_score = 0 steps = 0 loss = 0 for e in range(3000): done = False memory = Memory() score = 0 state = env.reset() state = torch.Tensor(state).to(device) state = state.unsqueeze(0) while not done: steps += 1 action = net.get_action(state) next_state, reward, done, _ = env.step(action) next_state = torch.Tensor(next_state) next_state = next_state.unsqueeze(0) mask = 0 if done else 1 reward = reward if not done or score == 499 else -1 action_one_hot = torch.zeros(2) action_one_hot[action] = 1 memory.push(state, next_state, action_one_hot, reward, mask) score += reward state = next_state loss = QNet.train_model(net, memory.sample(), optimizer) score = score if score == 500.0 else score + 1 running_score = 0.99 * running_score + 0.01 * score if e % log_interval == 0: print('{} episode | score: {:.2f}'.format(e, running_score)) writer.add_scalar('log/score', float(running_score), e) writer.add_scalar('log/loss', float(loss), e) if running_score > goal_score: break
def main(): if not (os.path.isdir("logs")): os.makedirs("logs") working_dir = "logs/" + args.dir if not (os.path.isdir(working_dir)): raise NameError(args.dir + " does not exist in dir logs") print(args) env = QubeSwingupEnv(use_simulator=args.sim, batch_size= 2048*4) num_inputs = env.observation_space.shape[0] num_actions = NUMBER_OF_ACTIONS print('state size:', num_inputs) print('action size:', num_actions) net = QNet(num_inputs, num_actions) if not args.new_net else QNet_more_layers(num_inputs, num_actions) net.load_state_dict(torch.load(working_dir + "/best_model.pth", map_location=torch.device(device))) net.to(device) net.eval() running_score = 0 epsilon = 1.0 steps = 0 beta = beta_start loss = 0 best_running_score = -1000 for e in range(1): done = False score = 0 state = env.reset() state = torch.Tensor(state).to(device) state = state.unsqueeze(0) while not done: steps += 1 action = get_continuous_action(get_action(state, net)) if np.abs(state[0][1].item()) < deg2rad(25): action = pd_control_policy(state.cpu().numpy()[0])[0] next_state, reward, done, info = env.step(action) reward = give_me_reward(info["alpha"], info["theta"]) if args.sim: env.render() reward = give_me_reward(info["alpha"], info["theta"]) if done: print(info) print("theta:" , info["theta"] * 180/np.pi) next_state = torch.Tensor(next_state).to(device) next_state = next_state.unsqueeze(0) score += reward state = next_state running_score = 0.99 * running_score + 0.01 * score print('{} episode | running_score: {:.2f} | score: {:.2f} | steps: {} '.format(e, running_score, score, steps)) env.close()
def __init__(self, env): super(SACV, self).__init__() self.env = env n_a = env.action_space.shape[0] n_s = env.observation_space.shape[0] pi_net = PiNet(n_s, n_a, distribution='Normal') self.pi_net = pi_net.to(self.device) q_net_1 = QNet(n_s, n_a) self.q_net_1 = q_net_1.to(self.device) q_net_2 = QNet(n_s, n_a) self.q_net_2 = q_net_2.to(self.device) v_net = QNet(n_s, 0) self.v_net = v_net.to(self.device) v_target = QNet(n_s, 0) self.v_target = v_target.to(self.device) self.load_state_dict(self.v_target, self.v_net.state_dict()) self.optimizer_q_1 = torch.optim.Adam(self.q_net_1.parameters(), lr=self.lr_q, betas=(0.9, 0.999), weight_decay=1e-2) self.optimizer_q_2 = torch.optim.Adam(self.q_net_2.parameters(), lr=self.lr_q, betas=(0.9, 0.999), weight_decay=1e-2) self.optimizer_v = torch.optim.Adam(self.v_net.parameters(), lr=self.lr_q, betas=(0.9, 0.999), weight_decay=1e-2) # eps = 1e-04, self.optimizer_p = torch.optim.Adam(self.pi_net.parameters(), lr=self.lr_p, betas=(0.9, 0.999), weight_decay=0) self.sample = self.actor_rb
def __init__(self, state_size, action_size, seed): """ Params ====== state_size (int): state dimension action_size (int): action dimension seed (int): random seed for replicating experiment """ self.state_size = state_size self.action_size = action_size self.seed = random.seed(seed) self.QNet_local = QNet(state_size, action_size, seed).to(device) self.QNet_target = QNet(state_size, action_size, seed).to(device) self.optimizer = optim.Adam(self.QNet_local.parameters(), lr=LR) self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed) self.t_step = 0
def __init__(self, actor_id, n_actors, shared_dict, device='cpu'): # params self.gamma = 0.99 self.epsilon = 0.4 ** (1 + actor_id * 7 / (n_actors - 1)) self.bootstrap_steps = 3 self.alpha = 0.6 self.priority_epsilon = 1e-6 self.device = device self.actor_id = actor_id # path self.memory_path = os.path.join( './', 'logs', 'memory') # memory self.memory_size = 50000 self.batch_size = 32 self.action_repeat = 4 self.n_stacks = 4 self.burn_in_length = 10 self.learning_length = 10 self.overlap_length = 10 self.eta = 0.9 self.sequence_length = self.burn_in_length + self.learning_length self.stack_count = self.n_stacks // self.action_repeat self.memory_save_interval = 5 self.episode_start_index = 0 self.n_steps_memory = NStepMemory(self.bootstrap_steps, self.gamma) self.replay_memory = ReplayMemory(self.memory_size, self.batch_size, self.bootstrap_steps) # net self.shared_dict = shared_dict self.net_load_interval = 5 self.net = QNet(self.device).to(self.device) self.target_net = QNet(self.device).to(self.device) self.target_net.load_state_dict(self.net.state_dict()) # env self.env = PongEnv(self.action_repeat, self.n_stacks) self.episode_reward = 0 self.n_episodes = 0 self.n_steps = 0 self.memory_count = 0 self.state = self.env.reset()
def main(): env = gym.make(env_name) env.seed(500) torch.manual_seed(500) num_inputs = env.observation_space.shape[0] num_actions = env.action_space.n print('state size:', num_inputs) print('action size:', num_actions) online_net = QNet(num_inputs, num_actions) target_net = QNet(num_inputs, num_actions) target_net.load_state_dict(online_net.state_dict()) online_net.share_memory() target_net.share_memory() optimizer = SharedAdam(online_net.parameters(), lr=lr) global_ep, global_ep_r, res_queue = mp.Value('i', 0), mp.Value('d', 0.), mp.Queue() writer = SummaryWriter('logs') online_net.to(device) target_net.to(device) online_net.train() target_net.train() workers = [ Worker(online_net, target_net, optimizer, global_ep, global_ep_r, res_queue, i) for i in range(mp.cpu_count()) ] [w.start() for w in workers] res = [] while True: r = res_queue.get() if r is not None: res.append(r) [ep, ep_r, loss] = r writer.add_scalar('log/score', float(ep_r), ep) writer.add_scalar('log/loss', float(loss), ep) else: break [w.join() for w in workers]
def __init__(self): super(Off_policy, self).__init__() self.memory = Replay_buffer(capacity=p.exploitory_policy_memory_size) self.exploratory_policy = GaussianPolicy( self.state_space, self.action_space).to(self.device) self.exploratory_Q = QNet(self.state_space, self.action_space).to(self.device) self.exploratory_Q_target = QNet(self.state_space, self.action_space).to(self.device) self.exploratory_policy_optim = Adam( self.exploratory_policy.parameters(), lr=p.lr) self.exploratory_Q_optim = Adam(self.exploratory_Q.parameters(), lr=p.lr) self.target_update(self.exploratory_policy, self.exploitory_policy, 1.0) self.kl_normalizer = Normalizer(1) self.ex_rewards_normalizer = Normalizer(1)
def main(): env = gym.make(args.env_name) env.seed(500) torch.manual_seed(500) img_shape = env.observation_space.shape num_actions = 3 print('image size:', img_shape) print('action size:', num_actions) net = QNet(num_actions) net.load_state_dict(torch.load(args.save_path + 'model.pth')) net.to(device) net.eval() epsilon = 0 for e in range(5): done = False score = 0 state = env.reset() state = pre_process(state) state = torch.Tensor(state).to(device) history = torch.stack((state, state, state, state)) for i in range(3): action = env.action_space.sample() state, reward, done, info = env.step(action) state = pre_process(state) state = torch.Tensor(state).to(device) state = state.unsqueeze(0) history = torch.cat((state, history[:-1]), dim=0) while not done: if args.render: env.render() steps += 1 qvalue = net(history.unsqueeze(0)) action = get_action(0, qvalue, num_actions) next_state, reward, done, info = env.step(action + 1) next_state = pre_process(next_state) next_state = torch.Tensor(next_state).to(device) next_state = next_state.unsqueeze(0) next_history = torch.cat((next_state, history[:-1]), dim=0) score += reward history = next_history print('{} episode | score: {:.2f}'.format(e, score))
def __init__(self, *largs, **kwargs): super(RBI, self).__init__(*largs, **kwargs) pi_net = PiNet(self.ns, self.na, distribution='Normal') self.pi_net = pi_net.to(self.device) pi_target = PiNet(self.ns, self.na, distribution='Normal') self.pi_target = pi_target.to(self.device) self.load_state_dict(self.pi_target, self.pi_net.state_dict()) q_net_1 = QNet(self.ns, self.na) self.q_net_1 = q_net_1.to(self.device) q_target_1 = QNet(self.ns, self.na) self.q_target_1 = q_target_1.to(self.device) self.load_state_dict(self.q_target_1, self.q_net_1.state_dict()) q_net_2 = QNet(self.ns, self.na) self.q_net_2 = q_net_2.to(self.device) q_target_2 = QNet(self.ns, self.na) self.q_target_2 = q_target_2.to(self.device) self.load_state_dict(self.q_target_2, self.q_net_2.state_dict()) self.optimizer_q_1 = torch.optim.Adam(self.q_net_1.parameters(), lr=self.lr_q, betas=(0.9, 0.999), weight_decay=self.weight_decay_q) self.optimizer_q_2 = torch.optim.Adam(self.q_net_2.parameters(), lr=self.lr_q, betas=(0.9, 0.999), weight_decay=self.weight_decay_q) self.optimizer_p = torch.optim.Adam(self.pi_net.parameters(), lr=self.lr_p, betas=(0.9, 0.999), weight_decay=self.weight_decay_p) self.alpha = self.rbi_alpha if self.entropy_tunning: # self.target_entropy = -float(self.na) std_target = 0.3 / math.sqrt(self.na) self.target_entropy = self.na * 0.5 * math.log(2 * math.pi * math.e * (std_target ** 2)) print(f'target entropy: {self.target_entropy}') self.lr_alpha = 0.01
def __init__(self, n_actors, device='cuda:0'): # params self.gamma = 0.99 self.alpha = 0.6 self.bootstrap_steps = 3 self.initial_exploration = 50000 self.priority_epsilon = 1e-6 self.device = device self.n_epochs = 0 self.n_actors = n_actors # path self.memory_path = os.path.join('./', 'logs', 'memory') self.net_path = os.path.join('./', 'logs', 'model', 'net.pt') self.target_net_path = os.path.join('./', 'logs', 'model', 'target_net.pt') # memory self.memory_size = 500000 self.batch_size = 128 self.memory_load_interval = 10 self.replay_memory = ReplayMemory(self.memory_size, self.batch_size, self.bootstrap_steps) # net self.net_save_interval = 50 self.target_update_interval = 1000 self.net = QNet(self.net_path, self.device).to(self.device) self.target_net = QNet(self.target_net_path, self.device).to(self.device) self.target_net.load_state_dict(self.net.state_dict()) self.net.save() self.target_net.save() self.optim = optim.RMSprop(self.net.parameters(), lr=0.00025 / 4.0, alpha=0.95, eps=1.5e-7, centered=True)
def __init__(self, *largs, **kwargs): super(TD3, self).__init__(*largs, **kwargs) pi_net = PiNet(self.ns, self.na) self.pi_net = pi_net.to(self.device) pi_target = PiNet(self.ns, self.na) self.pi_target = pi_target.to(self.device) self.load_state_dict(self.pi_target, self.pi_net.state_dict()) q_net_1 = QNet(self.ns, self.na) self.q_net_1 = q_net_1.to(self.device) q_target_1 = QNet(self.ns, self.na) self.q_target_1 = q_target_1.to(self.device) self.load_state_dict(self.q_target_1, self.q_net_1.state_dict()) q_net_2 = QNet(self.ns, self.na) self.q_net_2 = q_net_2.to(self.device) q_target_2 = QNet(self.ns, self.na) self.q_target_2 = q_target_2.to(self.device) self.load_state_dict(self.q_target_2, self.q_net_2.state_dict()) self.optimizer_q_1 = torch.optim.Adam(self.q_net_1.parameters(), lr=self.lr_q, betas=(0.9, 0.999)) self.optimizer_q_2 = torch.optim.Adam(self.q_net_2.parameters(), lr=self.lr_q, betas=(0.9, 0.999)) self.optimizer_p = torch.optim.Adam(self.pi_net.parameters(), lr=self.lr_p, betas=(0.9, 0.999)) self.noise = RandomNoise( torch.zeros(1, self.na).to(self.device), self.epsilon)
def __init__(self, run): self.run = run ckpt_dir = os.path.join(run, 'ckpt') ckpts = glob2.glob(os.path.join(ckpt_dir, '*.pth')) assert ckpts, "No checkpoints to resume from!" def get_epoch(ckpt_url): s = re.findall("ckpt_e(\d+).pth", ckpt_url) epoch = int(s[0]) if s else -1 return epoch, ckpt_url start_epoch, ckpt = max(get_epoch(c) for c in ckpts) print('Checkpoint:', ckpt) if torch.cuda.is_available(): model = QNet().cuda() else: model = QNet() ckpt = torch.load(ckpt) model.load_state_dict(ckpt['model']) model.eval() self.model = model
def __init__(self, input_size, action_size, gamma, tau, alpha, hidden_size, lr, device): self.gamma, self.tau, self.alpha = gamma, tau, alpha self.lr, self.device = lr, device self.policy = Actor(input_size, hidden_size, action_size).to(self.device) self.critic = QNet(input_size, hidden_size, action_size).to(self.device) self.policy_optim = torch.optim.Adam(self.policy.parameters(), lr=self.lr) self.critic_optim = torch.optim.Adam(self.critic.parameters(), lr=self.lr) self.critic_target = copy.deepcopy(self.critic) self.critic_target.requires_grad_(False)
def setup(self, obs_shape, nb_action): self.lr_coef = 1 self.epsilon = 1 self.nb_action = nb_action model_args = Singleton_arger()['model'] qnet = QNet(obs_shape, nb_action) self.qnet = copy.deepcopy(qnet) self.target_qnet = copy.deepcopy(qnet) self.memory = Memory(self.buffer_size, nb_action, self.with_cuda) if self.with_cuda: self.qnet.cuda() self.target_qnet.cuda() self.qnet_optim = Adam(self.qnet.parameters(), lr=self.critic_lr)
def __init__(self, *largs, **kwargs): super(PPO, self).__init__(*largs, **kwargs) self.pi_net = PiNet(self.ns, self.na, distribution='Normal', bounded=False, agent='ppo').to(self.device) self.v_net = QNet(self.ns, 0, agent='ppo').to(self.device) self.optimizer_v = torch.optim.Adam(self.v_net.parameters(), lr=self.lr_q, betas=(0.9, 0.999), weight_decay=self.weight_decay_q) self.optimizer_p = torch.optim.Adam(self.pi_net.parameters(), lr=self.lr_p, betas=(0.9, 0.999), weight_decay=self.weight_decay_p)
def main(): env = gym.make(args.env_name) env.seed(500) torch.manual_seed(500) num_inputs = env.observation_space.shape[0] num_actions = env.action_space.n print('state size:', num_inputs) print('action size:', num_actions) net = QNet(num_inputs, num_actions) net.load_state_dict(torch.load(args.save_path + 'model.pth')) net.to(device) net.eval() running_score = 0 steps = 0 for e in range(5): done = False score = 0 state = env.reset() state = torch.Tensor(state).to(device) state = state.unsqueeze(0) while not done: env.render() steps += 1 qvalue = net(state) action = get_action(qvalue) next_state, reward, done, _ = env.step(action) next_state = torch.Tensor(next_state).to(device) next_state = next_state.unsqueeze(0) score += reward state = next_state print('{} episode | score: {:.2f}'.format(e, score))
def main(): net = QNet().cuda().train() # print(net) optimizer = optim.SGD([{ 'params': [ param for name, param in net.named_parameters() if name[-4:] == 'bias' ], 'lr': 2 * args['lr'] }, { 'params': [ param for name, param in net.named_parameters() if name[-4:] != 'bias' ], 'lr': args['lr'], 'weight_decay': args['weight_decay'] }], momentum=args['momentum']) if len(args['snapshot']) > 0: print('training resumes from ' + args['snapshot']) net.load_state_dict( torch.load( os.path.join(ckpt_path, exp_name, args['snapshot'] + '.pth'))) optimizer.load_state_dict( torch.load( os.path.join(ckpt_path, exp_name, args['snapshot'] + '_optim.pth'))) optimizer.param_groups[0]['lr'] = 2 * args['lr'] optimizer.param_groups[1]['lr'] = args['lr'] check_mkdir(ckpt_path) check_mkdir(os.path.join(ckpt_path, exp_name)) open(log_path, 'w').write(str(args) + '\n\n') train(net, optimizer)
def main(): env = gym.make(args.env_name) env.seed(500) torch.manual_seed(500) state_size = env.observation_space.shape[0] action_size = env.action_space.n print('state size:', state_size) print('action size:', action_size) q_net = QNet(state_size, action_size, args) target_q_net = QNet(state_size, action_size, args) optimizer = optim.Adam(q_net.parameters(), lr=0.001) update_target_model(q_net, target_q_net) writer = SummaryWriter(args.logdir) replay_buffer = deque(maxlen=10000) running_score = 0 steps = 0 for episode in range(args.max_iter_num): done = False score = 0 state = env.reset() state = np.reshape(state, [1, state_size]) while not done: if args.render: env.render() steps += 1 q_values = q_net(torch.Tensor(state)) action = get_action(q_values, action_size, args.epsilon) next_state, reward, done, _ = env.step(action) next_state = np.reshape(next_state, [1, state_size]) reward = reward if not done or score == 499 else -1 mask = 0 if done else 1 replay_buffer.append((state, action, reward, next_state, mask)) state = next_state score += reward if steps > args.initial_exploration: args.epsilon -= args.epsilon_decay args.epsilon = max(args.epsilon, 0.1) mini_batch = random.sample(replay_buffer, args.batch_size) q_net.train(), target_q_net.train() train_model(q_net, target_q_net, optimizer, mini_batch) if steps % args.update_target == 0: update_target_model(q_net, target_q_net) score = score if score == 500.0 else score + 1 running_score = 0.99 * running_score + 0.01 * score if episode % args.log_interval == 0: print( '{} episode | running_score: {:.2f} | epsilon: {:.2f}'.format( episode, running_score, args.epsilon)) writer.add_scalar('log/score', float(score), episode) if running_score > args.goal_score: if not os.path.isdir(args.save_path): os.makedirs(args.save_path) ckpt_path = args.save_path + 'model.pth.tar' torch.save(q_net.state_dict(), ckpt_path) print('Running score exceeds 400. So end') break
if __name__ == "__main__": env = env.MinecraftEnv() env.init(allowContinuousMovement=["move", "turn"], videoResolution=[800, 600]) env.seed(500) torch.manual_seed(500) render_map = False num_inputs = env.observation_space.shape num_actions = len(env.action_names[0]) print('state size:', num_inputs) print('action size:', num_actions) model = QNet(num_actions) model.apply(weights_init) target_model = QNet(num_actions) update_target_model(model, target_model) model.train() target_model.train() optimizer = optim.Adam(model.parameters(), lr=hp.lr, weight_decay=hp.l2_rate) memory = Memory(100000) if render_map: root, canvas = init_map() steps = 0
def main(): if not (os.path.isdir("logs")): os.makedirs("logs") if (args.entropy and args.boltzmann): raise ValueError("Entropy as well as Boltzmann set.") print(args) working_dir = "logs/" + args.dir if not (os.path.isdir(working_dir)): os.mkdir(working_dir) env = QubeSwingupEnv(use_simulator=True) num_inputs = env.observation_space.shape[0] num_actions = NUMBER_OF_ACTIONS print('state size:', num_inputs) print('action size:', num_actions) online_net = QNet(num_inputs, num_actions) target_net = QNet(num_inputs, num_actions) update_target_model(online_net, target_net) optimizer = optim.Adam(online_net.parameters(), lr=lr) writer = SummaryWriter(working_dir) online_net.to(device) target_net.to(device) online_net.train() target_net.train() memory = Memory_With_TDError(replay_memory_capacity) running_score = 0 epsilon = 1.0 steps = 0 beta = beta_start loss = 0 training_started = False best_running_score = -1000 for e in range(args.e): done = False score = 0 state = env.reset() state = torch.Tensor(state).to(device) state = state.unsqueeze(0) start_time = time.time() while not done: steps += 1 action = get_action(state, target_net, epsilon, use_entropy=args.entropy, use_boltzmann=args.boltzmann) next_state, reward, done, info = env.step( get_continuous_action(action)) reward = give_me_reward(info["alpha"], info["theta"]) next_state = torch.Tensor(next_state).to(device) next_state = next_state.unsqueeze(0) mask = 0 if done else 1 action_one_hot = np.zeros(NUMBER_OF_ACTIONS) action_one_hot[action] = 1 memory.push(state, next_state, action_one_hot, reward, mask) score += reward state = next_state if steps > initial_exploration: if not training_started: print("---------------- training started ---------------") training_started = True epsilon -= 0.000005 epsilon = max(epsilon, 0.1) beta += 0.000005 beta = min(1, beta) batch, weights = memory.sample(batch_size, online_net, target_net, beta) loss = QNet.train_model(online_net, target_net, optimizer, batch, weights, device) if steps % update_target == 0: update_target_model(online_net, target_net) end_time = time.time() running_score = 0.99 * running_score + 0.01 * score if e % log_interval == 0: print( '{} episode | score: {:.2f} | epsilon: {:.2f} | beta: {:.2f}'. format(e, running_score, epsilon, beta)) writer.add_scalar('log/score', float(running_score), e) writer.add_scalar('log/loss', float(loss), e) if running_score > best_running_score and args.save: torch.save(online_net.state_dict(), working_dir + "/best_model.pth") best_running_score = running_score
def main(): # cartpole test if (cartpole_test): envs_fun = [lambda: gym.make('CartPole-v0')] envs_fun = np.tile(envs_fun, 3) envs = ShmemVecEnv(envs_fun) dummy_env = envs_fun[0]() else: INPUT_FILE = '../data/05f2a901.json' with open(INPUT_FILE, 'r') as f: puzzle = json.load(f) envs_fun = [ lambda: gym.make('arc-v0', input=task['input'], output=task['output'], need_ui=need_ui) for task in puzzle['train'] ] #pdb.set_trace() envs_fun = envs_fun[0:1] envs = ShmemVecEnv(envs_fun) dummy_env = envs_fun[0]() env_num = len(envs_fun) torch.manual_seed(500) num_inputs = dummy_env.observation_space.shape[0] num_actions = dummy_env.action_space.n print('state size:', num_inputs) print('action size:', num_actions) online_net = QNet(num_inputs, num_actions, cartpole_test, evalution_mode) target_net = QNet(num_inputs, num_actions, cartpole_test, evalution_mode) if (evalution_mode): online_net = torch.load('../result/arc0.model') target_net = torch.load('../result/arc0.model') update_target_model(online_net, target_net) optimizer = optim.Adam(online_net.parameters(), lr=lr) writer = SummaryWriter('logs') online_net.to(device) target_net.to(device) online_net.train() target_net.train() memory = Memory(replay_memory_capacity) score = 0 epsilon = 1.0 steps = 0 loss = 0 states = envs.reset() try: while True: if (need_ui): envs.render() steps += 1 global initial_exploration if (initial_exploration > 0): initial_exploration -= 1 actions = [] for state in states: state = torch.Tensor(state).to(device) state = state.unsqueeze(0) action = get_action(state, target_net, 0 if evalution_mode else epsilon, dummy_env) if (evalution_mode): print(action) actions.append(action) next_states, rewards, dones, info = envs.step(actions) #print(rewards) masks = np.zeros(envs.num_envs) for i in range(envs.num_envs): masks[i] = 0 if dones[i] else 1 for i in range(envs.num_envs): #print(rewards[i]) action_one_hot = np.zeros(dummy_env.action_space.n) action_one_hot[actions[i]] = 1 memory.push(states[i], next_states[i], action_one_hot, rewards[i], masks[i]) #score += reward states = next_states if not evalution_mode and steps > initial_exploration: epsilon -= 0.00003 epsilon = max(epsilon, 0.1) batch = memory.sample(batch_size) loss = QNet.train_model(online_net, target_net, optimizer, batch, device) if steps % update_target == 0: update_target_model(online_net, target_net) if (steps > 1028): states = envs.reset() steps = 0 print( 'new epsisode ------------------------------------------') except KeyboardInterrupt: print('save model') torch.save(target_net, '../result/arc.model') sys.exit(0)
def main(): env = gym.make(env_name) env.seed(500) torch.manual_seed(500) num_inputs = env.observation_space.shape[0] num_actions = env.action_space.n print('state size:', num_inputs) print('action size:', num_actions) online_net = QNet(num_inputs, num_actions) target_net = QNet(num_inputs, num_actions) update_target_model(online_net, target_net) optimizer = optim.Adam(online_net.parameters(), lr=lr) writer = SummaryWriter('logs') online_net.to(device) target_net.to(device) online_net.train() target_net.train() memory = Memory_With_TDError(replay_memory_capacity) running_score = 0 epsilon = 1.0 steps = 0 beta = beta_start loss = 0 for e in range(3000): done = False score = 0 state = env.reset() state = torch.Tensor(state).to(device) state = state.unsqueeze(0) while not done: steps += 1 action = get_action(state, target_net, epsilon, env) next_state, reward, done, _ = env.step(action) next_state = torch.Tensor(next_state) next_state = next_state.unsqueeze(0) mask = 0 if done else 1 reward = reward if not done or score == 499 else -1 action_one_hot = np.zeros(2) action_one_hot[action] = 1 memory.push(state, next_state, action_one_hot, reward, mask) score += reward state = next_state if steps > initial_exploration: epsilon -= 0.00005 epsilon = max(epsilon, 0.1) beta += 0.00005 beta = min(1, beta) batch, weights = memory.sample(batch_size, online_net, target_net, beta) loss = QNet.train_model(online_net, target_net, optimizer, batch, weights) if steps % update_target == 0: update_target_model(online_net, target_net) score = score if score == 500.0 else score + 1 running_score = 0.99 * running_score + 0.01 * score if e % log_interval == 0: print( '{} episode | score: {:.2f} | epsilon: {:.2f} | beta: {:.2f}'. format(e, running_score, epsilon, beta)) writer.add_scalar('log/score', float(running_score), e) writer.add_scalar('log/loss', float(loss), e) if running_score > goal_score: break
def main(): env = gym.make(env_name) env.seed(500) torch.manual_seed(500) ### NNのIn-Outは環境によって異なる num_inputs = env.observation_space.shape[0] num_actions = env.action_space.n print('state size:', num_inputs) print('action size:', num_actions) ### 2つのNWを作成・初期化 online_net = QNet(num_inputs, num_actions) target_net = QNet(num_inputs, num_actions) update_target_model(online_net, target_net) optimizer = optim.Adam(online_net.parameters(), lr=lr) ### 各NWの設定 CPU / GPU online_net.to(device) target_net.to(device) ### 各NWの設定 初めは学習モードにする online_net.train() target_net.train() ### 学習前の初期設定 memory = Memory(replay_memory_capacity) running_score = 0 epsilon = 1.0 steps = 0 loss = 0 steps_before = 0 for e in range(3000): done = False score = 0 state = env.reset() state = torch.Tensor(state).to(device) state = state.unsqueeze(0) while not done: steps += 1 ### 行動の決定はtarget_netで行う action = get_action(state, target_net, epsilon, env) ### 次の状態の観測、報酬の獲得 next_state, reward, done, _ = env.step(action) next_state = torch.Tensor(next_state) next_state = next_state.unsqueeze(0) if e % 10 == 0: print(next_state, action, reward) ### わかりにくいので書き変えた if done: mask = 0 else: mask = 1 ### memoryに記録 action_one_hot = np.zeros(num_actions) action_one_hot[action] = 1 memory.push(state, next_state, action_one_hot, reward, mask) ### rewardは基本的に-1 score += reward ### そのepisodeで何ステップ行ったかを記録するためだけのもの state = next_state if steps > initial_exploration: epsilon -= 0.00005 epsilon = max(epsilon, 0.1) ### online_net の学習 batch = memory.sample(batch_size) loss = QNet.train_model(online_net, target_net, optimizer, batch) ### たまにtarget_netをonline_netで上書きする if steps % update_target == 0: update_target_model(online_net, target_net) print("Ep {0:04d}: {1} step".format(e, steps - steps_before)) steps_before = steps score = score if score == 200.0 else score + 1 running_score = 0.99 * running_score + 0.01 * score if e % log_interval == 0: print('{} episode | score: {:.2f} | epsilon: {:.2f}'.format( e, running_score, epsilon)) if running_score > goal_score: break