def __init__(self, thread_index, global_network, initial_learning_rate, learning_rate_input, grad_applier, max_global_time_step, device, network_scope="network", scene_scope="scene", task_scope="task"): self.thread_index = thread_index self.learning_rate_input = learning_rate_input self.max_global_time_step = max_global_time_step self.network_scope = network_scope self.scene_scope = scene_scope self.task_scope = task_scope self.scopes = [network_scope, scene_scope, task_scope] self.local_network = ActorCriticFFNetwork(action_size=ACTION_SIZE, device=device, network_scope=network_scope, scene_scopes=[scene_scope]) self.local_network.prepare_loss(ENTROPY_BETA, self.scopes) self.trainer = AccumTrainer(device) self.trainer.prepare_minimize(self.local_network.total_loss, self.local_network.get_vars()) self.accum_gradients = self.trainer.accumulate_gradients() self.reset_gradients = self.trainer.reset_gradients() accum_grad_names = [ self._local_var_name(x) for x in self.trainer.get_accum_grad_list() ] global_net_vars = [ x for x in global_network.get_vars() if self._get_accum_grad_name(x) in accum_grad_names ] self.apply_gradients = grad_applier.apply_gradients( global_net_vars, self.trainer.get_accum_grad_list()) self.sync = self.local_network.sync_from(global_network) self.env = None self.local_t = 0 self.initial_learning_rate = initial_learning_rate self.episode_reward = 0 self.episode_length = 0 self.episode_max_q = -np.inf
def test(rank, scene_scope, task_scope, args, shared_model, counter): torch.manual_seed(args.seed + rank) env = Environment({ 'scene_name': scene_scope, 'terminal_state_id': int(task_scope) }) model = ActorCriticFFNetwork(ACTION_SIZE) model.eval() height, width, layers = env.observation.shape video = cv2.VideoWriter('video/' + task_scope + '.mp4',-1,1,(width,height)) env.reset() state = torch.from_numpy(env.s_t) reward_sum = 0 done = True start_time = time.time() # a quick hack to prevent the agent from stucking actions = deque(maxlen=100) episode_length = 0 img = cv2.cvtColor(env.observation, cv2.COLOR_BGR2RGB) video.write(img) for i in range(100): episode_length += 1 # Sync with the shared model if done: model.load_state_dict(shared_model.state_dict()) logit, value = model(env.s_t, env.target) prob = F.softmax(logit, dim=1) action = prob.max(1, keepdim=True)[1].data.numpy() env.step(action[0, 0]) env.update() img = cv2.cvtColor(env.observation, cv2.COLOR_BGR2RGB) video.write(img) reward = env.reward state = env.s_t done = env.terminal print(env.terminal_state_id, env.current_state_id) done = done or episode_length >= args.max_episode_length reward_sum += reward # a quick hack to prevent the agent from stucking actions.append(action[0, 0]) if actions.count(actions[0]) == actions.maxlen: done = True if done: print("Time {}, num steps {}, FPS {:.0f}, episode reward {}, episode length {}".format( time.strftime("%Hh %Mm %Ss", time.gmtime(time.time() - start_time)), counter.value, counter.value / (time.time() - start_time), reward_sum, episode_length)) reward_sum = 0 episode_length = 0 actions.clear() env.reset() state = env.s_t break state = torch.from_numpy(state) cv2.destroyAllWindows() video.release()
from constants import CHECKPOINT_DIR from constants import NUM_EVAL_EPISODES from constants import VERBOSE from constants import TASK_TYPE from constants import TASK_LIST if __name__ == '__main__': device = "/cpu:0" # use CPU for display tool network_scope = TASK_TYPE list_of_tasks = TASK_LIST scene_scopes = list_of_tasks.keys() global_network = ActorCriticFFNetwork(action_size=ACTION_SIZE, device=device, network_scope=network_scope, scene_scopes=scene_scopes) sess = tf.Session() init = tf.global_variables_initializer() sess.run(init) saver = tf.train.Saver(global_network.get_vars()) checkpoint = tf.train.get_checkpoint_state(CHECKPOINT_DIR) if checkpoint and checkpoint.model_checkpoint_path: saver.restore(sess, checkpoint.model_checkpoint_path) print("checkpoint loaded: {}".format(checkpoint.model_checkpoint_path)) else: print("Could not find old checkpoint")
def __init__(self, thread_index, global_network, global_discriminator, initial_learning_rate, learning_rate_input, grad_applier, grad_applier_discriminator, max_global_time_step, device, device2, network_scope="network", scene_scope="scene", task_scope="task"): self.thread_index = thread_index self.learning_rate_input = learning_rate_input self.max_global_time_step = max_global_time_step self.network_scope = network_scope self.network_scope_D = network_scope + "_d" self.scene_scope = scene_scope self.task_scope = task_scope self.scopes = [network_scope, scene_scope, task_scope] self.scopes_d = [self.network_scope_D, task_scope] self.local_network = ActorCriticFFNetwork(action_size=ACTION_SIZE, device=device, network_scope=network_scope, scene_scopes=[scene_scope]) self.local_network.prepare_loss(ENTROPY_BETA, self.scopes) self.trainer = AccumTrainer(device) self.trainer.prepare_minimize( self.local_network. total_loss, #getting the gradients of for the local network variablkes self.local_network.get_vars()) #This part is for the newly added PPO loss (we need to keep old and new update parameters) new_variable_list = self.local_network.get_vars() old_varaible_list = self.local_network.get_vars_old() #For the ppo loss begining of the each iteration we need to sync old with current self.old_new_sync = self.local_network.sync_curre_old() self.accum_gradients = self.trainer.accumulate_gradients( ) #This is to assign gradients self.reset_gradients = self.trainer.reset_gradients( ) #after applying the grads to variables we need to resent those variables accum_grad_names = [ self._local_var_name(x) for x in self.trainer.get_accum_grad_list() ] #get the name list of all the grad vars global_net_vars = [ x for x in global_network.get_vars() if self._get_accum_grad_name(x) in accum_grad_names ] #check whether the global_network vars are mentioned in gradiet computations for them local_net_vars = [ x for x in self.local_network.get_vars() if self._get_accum_grad_name(x) in accum_grad_names ] self.apply_gradients = grad_applier.apply_gradients( global_net_vars, self.trainer.get_accum_grad_list()) self.apply_gradients_local = grad_applier.apply_gradients_local_net( local_net_vars, self.trainer.get_accum_grad_list()) #If this is unstable it is desireable to first apply the gradients on the local network and then clip and after that we apply self.sync = self.local_network.sync_from( global_network ) #this is to sync from the glocal network Apply updated global params to the local network #This part is for the Discriminator ######################################################################################### # self.local_discriminator = Discriminator_WGAN( # action_size=ACTION_SIZE, # device=device, # network_scope=network_scope, # scene_scopes=[scene_scope]) # # self.local_discriminator.prepare_loss_D(ENTROPY_BETA, self.scopes_d) # # self.trainer_D = AccumTrainer_d(device=device, name="AccumTrainer_d") # # self.trainer_D.prepare_minimize( self.local_discriminator.total_loss_d, # self.local_discriminator.get_vars()) # # # self.accum_gradients_d = self.trainer_D.accumulate_gradients() # self.reset_gradients_d = self.trainer_D.reset_gradients() # # accum_grad_names_discrimi = [ self._local_var_name(x) for x in self.trainer_D.get_accum_grad_list() ] # # global_discri_vars = [ x for x in global_discriminator.get_vars() if self._get_accum_grad_name(x) in accum_grad_names_discrimi ] local_discri_vars = [ x for x in self.local_discriminator.get_vars() if self._get_accum_grad_name(x) in accum_grad_names_discrimi ] # self.apply_gradients_discriminator = grad_applier_discriminator.apply_gradients( local_discri_vars, self.trainer_D.get_accum_grad_list() ) #applying grad to the LOCAL network # self.clip_local_d_weights = self.local_discriminator.clip_weights( ) #here we are clipping the global net weights directly. # self.sync_discriminator_l_G = self.local_discriminator.sync_to( global_discriminator) # self.sync_discriminator_G_l = self.local_discriminator.sync_from( global_discriminator) # self.D_var_G = global_discriminator.get_vars() self.D_var_l = self.local_discriminator.get_vars() # # # ######################################################################################### self.env = None self.local_t = 0 self.initial_learning_rate = initial_learning_rate self.episode_reward = 0 self.episode_length = 0 self.episode_max_q = -np.inf
def train(rank, scene_scope, task_scope, args, shared_model, counter, lock, optimizer=None): torch.manual_seed(args.seed + rank) #env = create_atari_env(args.env_name) #env.seed(args.seed + rank) env = Environment({ 'scene_name': scene_scope, 'terminal_state_id': int(task_scope) }) model = ActorCriticFFNetwork(ACTION_SIZE) if optimizer is None: # TODO: Discount learning rate based on episode length optimizer = my_optim.SharedRMSprop(shared_model.parameters(), lr=args.lr, alpha=args.alpha, eps=args.eps) optimizer.share_memory() model.train() env.reset() state = torch.from_numpy(env.s_t) done = True episode_length = 0 for i in range(int(args.max_episode_length)): # Sync with the shared model model.load_state_dict(shared_model.state_dict()) ''' if done: cx = Variable(torch.zeros(1, 256)) hx = Variable(torch.zeros(1, 256)) else: cx = Variable(cx.data) hx = Variable(hx.data) ''' values = [] log_probs = [] rewards = [] entropies = [] for step in range(args.num_steps): print('Thread: ', rank, ', step: ', step, 'epochs:', i) episode_length += 1 logit, value = model(env.s_t, env.target) prob = F.softmax(logit, dim=1) log_prob = F.log_softmax(logit, dim=1) entropy = -(log_prob * prob).sum(1, keepdim=True) entropies.append(entropy) action = prob.multinomial(num_samples=1).data log_prob = log_prob.gather(1, Variable(action)) env.step(action) #state, reward, done, _ = env.step(action.numpy()) env.update() state = env.s_t reward = env.reward done = env.terminal done = done or episode_length >= args.max_episode_length reward = max(min(reward, 1), -1) with lock: if counter.value % 1000 == 0: print('Now saving data. Please wait.') torch.save(shared_model.state_dict(), CHECKPOINT_DIR + '/' + 'checkpoint.pth.tar') counter.value += 1 if done: episode_length = 0 if env.terminal: print('Task completed') counter.value += 1 if done: episode_length = 0 env.reset() state = env.s_t state = torch.from_numpy(state) values.append(value) log_probs.append(log_prob) rewards.append(reward) if done: break R = torch.zeros(1, 1) if not done: _, value = model(env.s_t, env.target) R = value.data values.append(Variable(R)) policy_loss = 0 value_loss = 0 R = Variable(R) gae = torch.zeros(1, 1) for i in reversed(range(len(rewards))): R = args.gamma * R + rewards[i] advantage = R - values[i] value_loss = value_loss + 0.5 * advantage.pow(2) # Generalized Advantage Estimataion delta_t = rewards[i] + args.gamma * \ values[i + 1].data - values[i].data gae = gae * args.gamma * args.tau + delta_t policy_loss = policy_loss - \ log_probs[i] * Variable(gae) - args.entropy_coef * entropies[i] optimizer.zero_grad() (policy_loss + args.value_loss_coef * value_loss).backward() torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) ensure_shared_grads(model, shared_model) optimizer.step()
help='environment to train on (default: bathroom_02)') parser.add_argument('--no_shared', default=False, help='environment to train on (default: bathroom_02)') if __name__ == '__main__': torch.multiprocessing.set_start_method('spawn') os.environ['OMP_NUM_THREADS'] = '1' os.environ['CUDA_VISIBLE_DEVICES'] = "" list_of_tasks = TASK_LIST scene_scopes = list_of_tasks.keys() args = parser.parse_args() torch.manual_seed(args.seed) #env = create_atari_env(args.env_name) shared_model = ActorCriticFFNetwork(ACTION_SIZE) shared_model.share_memory() if args.no_shared: optimizer = None else: optimizer = my_optim.SharedRMSprop(shared_model.parameters(), lr=args.lr, alpha=args.alpha, eps=args.eps) optimizer.share_memory() processes = [] counter = mp.Value('i', 0) lock = mp.Lock()
def main(): # disable all v2 behavior tf.disable_v2_behavior() tf.disable_eager_execution() device = "/cpu:0" # use CPU for display tool network_scope = TASK_TYPE # Always 'navigation' list_of_tasks = TASK_LIST scene_scopes = list_of_tasks.keys() global_network = ActorCriticFFNetwork(action_size=ACTION_SIZE, device=device, network_scope=network_scope, scene_scopes=scene_scopes) sess = tf.Session() # sess = tf.coSession() init = tf.global_variables_initializer() sess.run(init) saver = tf.train.Saver() checkpoint = tf.train.get_checkpoint_state(CHECKPOINT_DIR) # see if we saved a checkpoint from past training? if checkpoint and checkpoint.model_checkpoint_path: saver.restore(sess, checkpoint.model_checkpoint_path) print("checkpoint loaded: {}".format(checkpoint.model_checkpoint_path)) else: print("Could not find old checkpoint") scene_stats = dict() for scene_scope in scene_scopes: # TODO: remove scene_scope = "FloorPlan402" scene_stats[scene_scope] = [] for task_scope in list_of_tasks[scene_scope]: # tasks are positions!!! # env = ai2thor.controller.Controller(scene="FloorPlan227", gridSize=0.25, width=1000, height=1000) with open(GOAL_FILE, 'r') as f: GOAL_DATA = json.load(f) GOAL_POS = GOAL_DATA["agent_position"] env = RLController({ 'scene': scene_scope, 'terminal_state_id': int(task_scope), 'goal_pos': GOAL_POS, 'goal_image_fpath': "data/FP402_goal_towel.png" }) env.docker_enabled = True ep_rewards = [] ep_lengths = [] ep_collisions = [] scopes = [network_scope, scene_scope] for i_episode in range(NUM_EVAL_EPISODES): env.reset() terminal = False ep_reward = 0 ep_collision = 0 ep_t = 0 while not terminal: # mirrors actions taken in paper # NOTE: rearranged these to mirror code in scene_loader list_of_actions = [ "MoveAhead", "RotateRight", "RotateLeft", "MoveBack" ] pi_values = global_network.run_policy( sess, env.curr_state, env.target, scopes) # action returned is an integer -- critical that the list_of_actions is in correct order action = sample_action(pi_values) print( "Ep_t: {} \n\tCollided?: {} \n\tAction: {} \n\tValue: {} \n\tAll Action Values: {}" .format(ep_t, env.collided, list_of_actions[action], pi_values[action], pi_values)) env.step(list_of_actions[action]) env.update() terminal = env.terminal if ep_t == 10000: break if env.collided: ep_collision += 1 ep_reward += env.reward ep_t += 1 ep_lengths.append(ep_t) ep_rewards.append(ep_reward) ep_collisions.append(ep_collision) print('evaluation: %s %s' % (scene_scope, task_scope)) print('mean episode reward: %.2f' % np.mean(ep_rewards)) print('mean episode length: %.2f' % np.mean(ep_lengths)) print('mean episode collision: %.2f' % np.mean(ep_collisions))