示例#1
0
文件: actor.py 项目: wotmd5731/J_LAB
class Actor:
    def __init__(self, actor_id,config,dev,shared_state,shared_queue,eps):
#        self.env = suite.load(domain_name="walker", task_name="run")
#        self.action_size = self.env.action_spec().shape[0]
#        self.obs_size = get_obs(self.env.reset().observation).shape[1]

        self.env = env_cover(config,dev)
        self.num_env = config['num_envs']
        self.shared_queue = shared_queue
        self.shared_state = shared_state
        self.dev = dev 

        self.actor_id = actor_id
        self.burn_in_length = config['burn_in_length'] # 40-80
        self.learning_length = config['learning_length']
        self.sequence_length = self.burn_in_length + self.learning_length
        self.n_step = config['n_step']
        self.sequence = []
        self.recurrent_state = []
        self.priority = []
        self.td_loss = deque(maxlen=self.learning_length)
#        self.memory_sequence_size = 1000
#        self.memory = ReplayMemory(memory_sequence_size=self.memory_sequence_size)
#        self.memory_save_interval = 3
        self.max_frame = config['actor_max_frame']
        self.gamma = config['gamma']
#        self.actor_parameter_update_interval = config['actor_parameter_update_interval']
        self.max_shared_q_size=config['max_shared_q_size']
        
        self.model_path = './'
        self.memory_path = './'
        
        self.actor = ActorNet(dev,config).to(self.dev)
        self.target_actor = ActorNet(dev,config).to(self.dev)
        self.critic = CriticNet(dev,config).to(self.dev)
        self.target_critic = CriticNet(dev,config).to(self.dev)
        
        self.actor.load_state_dict(self.shared_state["actor"].state_dict())
        self.target_actor.load_state_dict(self.shared_state["target_actor"].state_dict())
        self.critic.load_state_dict(self.shared_state["critic"].state_dict())
        self.target_critic.load_state_dict(self.shared_state["target_critic"].state_dict())

#        self.actor.load_state_dict(self.shared_state["actor"])
#        self.target_actor.load_state_dict(self.shared_state["target_actor"])
#        self.critic.load_state_dict(self.shared_state["critic"])
#        self.target_critic.load_state_dict(self.shared_state["target_critic"])
        self.action_argmax = config['action_argmax']
        
#        self.load_model()
        self.epsilon = eps 
    def __del__(self):
        self.env.close()
        
    def PrePro(self,obs):
        return obs
#        return torch.from_numpy(obs).detach().float().reshape((1,self.obs_size)).to(self.dev)
   
    def save_memory(self):
        
        model_dict = {'sequence': self.sequence,
                      'recurrent_state': self.recurrent_state,
                      'priority': self.priority,
                      }
        
        torch.save(model_dict, self.memory_path + 'memory.pt')
    
    
#    with open('outfile', 'wb') as fp:
#    pickle.dump(itemlist, fp)
#    
#    with open ('outfile', 'rb') as fp:
#    itemlist = pickle.load(fp)
    
    
    def load_model(self):
        if os.path.isfile(self.model_path + 'model.pt'):
            while True:
                try:
                    # TODO: Delete
#                    self.actor = ActorNet(self.obs_size, self.action_size, self.actor_id%2+1).cuda().eval()
#                    self.target_actor = deepcopy(self.actor)
#                    self.critic = CriticNet(self.obs_size, self.action_size, self.actor_id%2+1).cuda().eval()
#                    self.target_critic = deepcopy(self.critic)
                    #model_dict = torch.load(self.model_path + 'model.pt', map_location={'cuda:0':'cuda:{}'.format(self.actor_id%2+1)})
                    print('waiting  model.pt')
                    model_dict = torch.load(self.model_path + 'model.pt')
                    self.actor.load_state_dict(model_dict['actor'])
                    self.target_actor.load_state_dict(model_dict['target_actor'])
                    self.critic.load_state_dict(model_dict['critic'])
                    self.target_critic.load_state_dict(model_dict['target_critic'])
                    self.actor.to(self.dev)
                    self.target_actor.to(self.dev)
                    self.critic.to(self.dev)
                    self.target_critic.to(self.dev)
                    
                except:
                    sleep(np.random.rand() * 5 + 2)
                else:
                    break

    def calc_nstep_reward(self):
        for i in range(len(self.sequence) - self.n_step):
            self.sequence[i][2] = sum([ self.sequence[i+j][2] * (self.sequence[i+j][3] ** j) for j in range(self.n_step)] )
            

    def calc_priorities(self):
        with torch.no_grad():
            self.actor.reset_state()
            self.critic.reset_state()
            self.target_actor.reset_state()
            self.target_critic.reset_state()
#            self.td_loss = deque(maxlen=self.learning_length)
            self.td_loss = []
            self.priority = []
    
    #       이부분은  target 넷을  nstep 만큼 진행 해놓는것.
            for i in range(self.n_step):
                next_obs = self.sequence[i][0]
                next_action = self.target_actor(self.PrePro(next_obs)).to(self.dev)
                next_q_value = self.target_critic(self.PrePro(next_obs), next_action)
    
    #       n 스텝 진행 하면서 Q 벨류 예측.   seq[시퀀스][0:staet ,1:action ,2:reward,3:term->gamma]
            for i in range(len(self.sequence) - self.n_step):
    #            obs = torch.from_numpy(self.sequence[i][0]).unsqueeze(0)
                obs = self.sequence[i][0]
    #            action = self.sequence[i][1].unsqueeze(0)
                next_obs = self.sequence[i + self.n_step][0]

                action = self.sequence[i][1]
#                action = torch.Tensor(self.sequence[i][1]).view(1,-1).to(self.dev)
    #            next_obs = torch.from_numpy(self.sequence[i + self.n_step][0]).unsqueeze(0)
                next_action = self.target_actor(self.PrePro(next_obs)).to(self.dev)
    
                q_value = self.critic(self.PrePro(obs), action)
                reward = self.sequence[i][2]
                gamma = self.sequence[i + self.n_step - 1][3]
                next_q_value = self.target_critic(self.PrePro(next_obs), next_action)
                
                if i >= self.burn_in_length:
                    target_q_value = (reward + (gamma ** self.n_step)) * next_q_value
#                    target_q_value = invertical_vf(target_q_value)
                    self.td_loss.append(((q_value - target_q_value)**2))
                    if len(self.td_loss) > self.learning_length:
                        self.td_loss.pop(0)

                if i >= self.sequence_length:
                    self.priority.append(calc_priority(self.td_loss))
            
            
            
    
    def run(self):
#        sleep(random.random()*1)
        frame = 0
        if self.actor_id%3 == 0:
            win_r = vis.line(Y=torch.Tensor([0]), opts=dict(title ='reward'+str(self.epsilon)))
        reward_sum = 0
        
        
        while frame  < self.max_frame:
#            self.shared_state['frame'][self.actor_id]=frame
#            while self.shared_state['sleep'][self.actor_id] :
#                sleep(0.5)
            
            st, rt, dt  = self.env.reset()
            
            self.actor.reset_state()
            self.critic.reset_state()
            self.target_actor.reset_state()
            self.target_critic.reset_state()
            
            self.sequence = []
            self.recurrent_state = []
            self.priority = []
            
            self.td_loss.clear()
            if self.actor_id%3 == 0:
                win_r = vis.line(X=torch.Tensor([frame]), Y=torch.Tensor([reward_sum]), win= win_r , update ='append')
            
            reward_sum = 0
            count_step = 0     
            sleep(0.01)
            while sum(dt)!=self.num_env:
                
                frame+=1
                # get recurrent state
                actor_hx, actor_cx = self.actor.get_state()
                target_actor_hx, target_actor_cx = self.target_actor.get_state()
                critic_hx, critic_cx = self.critic.get_state()
                target_critic_hx, target_critic_cx = self.target_critic.get_state()
                
                action = self.actor(self.PrePro(st))
                target_action = self.target_actor(self.PrePro(st))
                _ = self.critic(self.PrePro(st), action)
                _ = self.target_critic(self.PrePro(st), target_action)

                noise = torch.normal(mean=torch.zeros([self.num_env,1]),std=torch.ones([self.num_env,1])).to(self.dev)
#                action = action.detach().item() +  np.random.normal(0, self.epsilon, (self.action_size))
#                action = np.clip(action, -1, 1)
                
                if self.action_argmax:
                    act = action.argmax(1).cpu().numpy().item()
                else:
                    action = action.cpu().numpy()
                if random.random()> self.epsilon:
                    act = random.randint(0,1)
#                action = (action+noise*self.epsilon).clamp(min=-1,max=1)

                st_1, rt, dt = self.env.step(act)
    
                reward_sum += rt
                count_step += 1
                gamma = torch.ones([self.num_env,1]).to(self.dev)*self.gamma*(1-dt)
#                gamma = self.gamma if not dt else 0.
                self.sequence.append([st, action, rt, gamma])
                st = st_1

                self.recurrent_state.append([torch.cat([actor_hx, actor_cx]), torch.cat([target_actor_hx, target_actor_cx]), 
                                                torch.cat([critic_hx, critic_cx]), torch.cat([target_critic_hx, target_critic_cx])])

#                if True:
                if self.shared_state["update"][self.actor_id]:
                    
                    
                    self.actor.load_state_dict(self.shared_state["actor"].state_dict())
                    self.target_actor.load_state_dict(self.shared_state["target_actor"].state_dict())
                    self.critic.load_state_dict(self.shared_state["critic"].state_dict())
                    self.target_critic.load_state_dict(self.shared_state["target_critic"].state_dict())
                    self.shared_state["update"][self.actor_id]=False
                    print('actor_update',self.actor.policy_l0.weight.data[0][0])
#                    self.load_model()


            if len(self.sequence) >= self.sequence_length:
                #self.sequence.extend([(st, action, 0., 0.) for i in range(self.n_step)])
                st, rt, dt = self.env.end_dummy()
                self.sequence.extend([[st,action, rt, dt] for i in range(self.n_step)])

                self.calc_nstep_reward()
                self.calc_priorities()
                
                
                for i in range(len(self.sequence)):
                    for j in range(4):
                        self.sequence[i][j] = self.sequence[i][j].cpu()
                for i in range(len(self.recurrent_state)):
                    for j in range(4):
                        self.recurrent_state[i][j] = self.recurrent_state[i][j].cpu()
                for i in range(len(self.priority)):
                    self.priority[i] = self.priority[i].cpu()
                blocking = True if self.shared_queue.qsize()>self.max_shared_q_size else False
                self.shared_queue.put([self.sequence, self.recurrent_state, self.priority],block=blocking)
                
#            if self.actor_id == 0:
            print('#',self.actor_id,'frame:', frame,'step:', count_step, 'reward:', reward_sum)
示例#2
0
class Actor:
    def __init__(self, actor_id):
        self.env = suite.load(domain_name="walker", task_name="run")
        self.action_size = self.env.action_spec().shape[0]
        self.obs_size = get_obs(self.env.reset().observation).shape[1]

        self.actor_id = actor_id
        self.burn_in_length = 20  # 40-80
        self.learning_length = 40
        self.sequence_length = self.burn_in_length + self.learning_length
        self.n_step = 5
        self.sequence = []
        self.recurrent_state = []
        self.priority = []
        self.td_loss = deque(maxlen=self.learning_length)
        self.memory_sequence_size = 1000
        self.memory = ReplayMemory(
            memory_sequence_size=self.memory_sequence_size)
        self.memory_save_interval = 3

        self.gamma = 0.997
        self.actor_parameter_update_interval = 500
        self.model_path = './model_data/'
        self.actor = ActorNet(self.obs_size,
                              self.action_size,
                              cuda_id=self.actor_id % 2 +
                              1).cuda(self.actor_id % 2 + 1).eval()
        self.target_actor = deepcopy(self.actor)
        self.critic = CriticNet(self.obs_size,
                                self.action_size,
                                cuda_id=self.actor_id % 2 +
                                1).cuda(self.actor_id % 2 + 1).eval()
        self.target_critic = deepcopy(self.critic)
        self.load_model()
        self.epsilon = 1
        self.last_obs = None

    def load_model(self):
        if os.path.isfile(self.model_path + 'model.pt'):
            while True:
                try:
                    # TODO: Delete
                    self.actor = ActorNet(self.obs_size, self.action_size,
                                          self.actor_id % 2 + 1).cuda().eval()
                    self.target_actor = deepcopy(self.actor)
                    self.critic = CriticNet(self.obs_size, self.action_size,
                                            self.actor_id % 2 +
                                            1).cuda().eval()
                    self.target_critic = deepcopy(self.critic)
                    #model_dict = torch.load(self.model_path + 'model.pt', map_location={'cuda:0':'cuda:{}'.format(self.actor_id%2+1)})
                    model_dict = torch.load(self.model_path + 'model.pt')
                    self.actor.load_state_dict(model_dict['actor'])
                    self.target_actor.load_state_dict(
                        model_dict['target_actor'])
                    self.critic.load_state_dict(model_dict['critic'])
                    self.target_critic.load_state_dict(
                        model_dict['target_critic'])
                    self.actor.cuda(self.actor_id % 2 + 1)
                    self.target_actor.cuda(self.actor_id % 2 + 1)
                    self.critic.cuda(self.actor_id % 2 + 1)
                    self.target_critic.cuda(self.actor_id % 2 + 1)
                except:
                    sleep(np.random.rand() * 5 + 2)
                else:
                    break

    def calc_nstep_reward(self):
        for i in range(len(self.sequence) - self.n_step):
            self.sequence[i][2][0] = sum([
                self.sequence[i + j][2][0] * (self.gamma**j)
                for j in range(self.n_step)
            ])

    def calc_priorities(self):
        self.actor.reset_state()
        self.critic.reset_state()
        self.target_actor.reset_state()
        self.target_critic.reset_state()
        self.td_loss = deque(maxlen=self.learning_length)
        self.priority = []

        for i in range(self.n_step):
            next_obs = torch.from_numpy(
                self.sequence[i][0]).cuda(self.actor_id % 2 + 1).unsqueeze(0)
            next_action = self.target_actor(next_obs)
            next_q_value = self.target_critic(
                next_obs, next_action).detach().cpu().numpy()

        for i in range(len(self.sequence) - self.n_step):
            obs = torch.from_numpy(
                self.sequence[i][0]).cuda(self.actor_id % 2 + 1).unsqueeze(0)
            action = torch.from_numpy(
                self.sequence[i][1]).cuda(self.actor_id % 2 + 1).unsqueeze(0)
            next_obs = torch.from_numpy(
                self.sequence[i + self.n_step][0]).cuda(self.actor_id % 2 +
                                                        1).unsqueeze(0)
            next_action = self.target_actor(next_obs)

            q_value = self.critic(obs, action).detach().cpu().numpy()
            reward = self.sequence[i][2][0]
            terminal = self.sequence[i + self.n_step - 1][3][0]
            next_q_value = self.target_critic(
                next_obs, next_action).detach().cpu().numpy()

            if i >= self.burn_in_length:
                target_q_value = (reward + (self.gamma**self.n_step) *
                                  (1. - terminal) * next_q_value)
                target_q_value = invertical_vf(
                    torch.tensor(target_q_value).cuda(
                        self.actor_id % 2 + 1)).detach().cpu().numpy()
                self.td_loss.append((q_value - target_q_value).mean())
            if i >= self.sequence_length:
                self.priority.append(
                    calc_priority(
                        np.array(list(self.td_loss), dtype=np.float32)**2.))

    def run(self):
        episode = 0
        step = 0
        reward_sum = 0

        while True:
            time_step = self.env.reset()
            obs = get_obs(time_step.observation)
            self.actor.reset_state()
            self.critic.reset_state()
            self.target_actor.reset_state()
            self.target_critic.reset_state()
            self.sequence = []
            self.recurrent_state = []
            self.priority = []
            self.td_loss.clear()
            last_obs = None
            episode_step = 0
            done = False
            if self.actor_id == 0 and episode != 0:
                print('episode:', episode, 'step:', step, 'reward:',
                      reward_sum)
            episode += 1
            reward_sum = 0

            while not time_step.last():

                # get recurrent state
                actor_hx, actor_cx = self.actor.get_state()
                target_actor_hx, target_actor_cx = self.target_actor.get_state(
                )
                critic_hx, critic_cx = self.critic.get_state()
                target_critic_hx, target_critic_cx = self.target_critic.get_state(
                )

                action = self.actor(
                    torch.from_numpy(obs).cuda(self.actor_id % 2 + 1))
                target_action = self.target_actor(
                    torch.from_numpy(obs).cuda(self.actor_id % 2 + 1))
                _ = self.critic(
                    torch.from_numpy(obs).cuda(self.actor_id % 2 + 1), action)
                _ = self.target_critic(
                    torch.from_numpy(obs).cuda(self.actor_id % 2 + 1),
                    target_action)

                action = action.detach().cpu().numpy()[0]
                action += np.random.normal(0, 0.3, (self.action_size))
                action = np.clip(action, -1, 1)

                reward = 0.
                sleep(0.01)
                for i in range(4):
                    time_step = self.env.step(action)
                    next_obs = get_obs(time_step.observation)
                    reward += time_step.reward
                    if time_step.last():
                        break

                reward_sum += reward
                step += 1
                episode_step += 1
                terminal = 1. if time_step.last() else 0.
                self.sequence.append((obs[0], action, [reward], [terminal]))
                obs = next_obs.copy()

                self.recurrent_state.append(
                    [[actor_hx[0], actor_cx[0]],
                     [target_actor_hx[0], target_actor_cx[0]],
                     [critic_hx[0], critic_cx[0]],
                     [target_critic_hx[0], target_critic_cx[0]]])

                if step % self.actor_parameter_update_interval == 0:
                    self.load_model()

            if len(self.sequence) >= self.sequence_length:
                self.sequence.extend([(np.zeros((self.obs_size),
                                                dtype=np.float32),
                                       np.zeros((self.action_size),
                                                dtype=np.float32), [0.], [1.])
                                      for i in range(self.n_step)])
                self.calc_nstep_reward()
                self.calc_priorities()
                self.memory.add(self.sequence, self.recurrent_state,
                                self.priority)

            if len(self.memory.memory) > self.memory_save_interval:
                self.memory.save(self.actor_id)