def test(config, env): ob_space = env.observation_space ac_space = env.action_space tf.reset_default_graph() gpu_opts = tf.GPUOptions(allow_growth=True) tf_config = tf.ConfigProto( inter_op_parallelism_threads=1, intra_op_parallelism_threads=1, gpu_options=gpu_opts, ) with tf.Session(config=tf_config) as sess: config.batch_size = 2 config.number_of_steps = 2 policy = build_policy(env, 'cnn') model = Model(policy=policy, env=env, nsteps=config.number_of_steps, ent_coef=config.entropy_weight, vf_coef=config.critic_weight, max_grad_norm=config.max_grad_norm, lr=config.learning_rate, alpha=config.rmsp_decay, epsilon=config.discount_factor, total_timesteps=config.timesteps, lrschedule='linear') model.load(config.load_path) return make_rollouts(config, env, model)
def enjoy(env_id, seed, policy, model_filename, fps=100): if policy == 'cnn': policy_fn = CnnPolicy elif policy == 'lstm': policy_fn = LstmPolicy elif policy == 'lnlstm': policy_fn = LnLstmPolicy env = wrap_deepmind(make_atari(env_id), clip_rewards=False, frame_stack=True) env.seed(seed) tf.reset_default_graph() ob_space = env.observation_space ac_space = env.action_space nsteps = 5 # default value, change if needed model = Model(policy=policy_fn, ob_space=ob_space, ac_space=ac_space, nenvs=1, nsteps=nsteps) model.load(model_filename) while True: obs, done = env.reset(), False episode_rew = 0 while not done: env.render() time.sleep(1.0 / fps) action, _, _, _ = model.step_model.step([obs.__array__()]) obs, rew, done, _ = env.step(action) episode_rew += rew print('Episode reward:', episode_rew) env.close()
def enjoy(env_path, seed, max_steps, base_port, model_path): env = _wrap_unity_env(env_path, seed, base_port, rank=1) model = Model(policy=CnnPolicy, ob_space=env.observation_space, ac_space=env.action_space, nenvs=1, nsteps=5) model.load( model_path ) # This will cause an unknown error when loading a model trained with 'learn' step_count = 0 while step_count <= max_steps: obs, done = env.reset(), False epsiode_rew = 0 step_count += 1 while not done: if keyboard.is_pressed('n'): break action, _, _, _ = model.step_model.step([obs.__array__()]) obs, rew, done, _ = env.step(action) epsiode_rew += rew print('Episode reward: ', episode_rew) print('Step Count: ', step_count) env.close()
def main(): env = gym.make("gridworld-v0") policy = CnnPolicy nsteps = 5 total_timesteps = int(80e6) vf_coef = 0.5 ent_coef = 0.01 max_grad_norm = 0.5 lr = 7e-4 lrschedule = 'linear' epsilon = 1e-5 alpha = 0.99 gamma = 0.99 log_interval = 100 ob_space = env.observation_space ac_space = env.action_space nenvs = env.num_envs #with tf.Session() as sess: with tf.Graph().as_default(), tf.Session().as_default(): #model = a2c.learn(policy=CnnPolicy, env=env, total_timesteps=int(0), seed=0) model = Model(policy=policy, ob_space=ob_space, ac_space=ac_space, nenvs=nenvs, nsteps=nsteps, ent_coef=ent_coef, vf_coef=vf_coef, max_grad_norm=max_grad_norm, lr=lr, alpha=alpha, epsilon=epsilon, total_timesteps=total_timesteps, lrschedule=lrschedule) model.load( "/Users/constantinos/Documents/Projects/cmu_gridworld/cmu_gym/a2c_open.pkl" ) while True: obs, done = env.reset(), False episode_rew = 0 while not done: env.render() obs, rew, done, _ = env.step(model(obs[None])[0]) episode_rew += rew print("Episode reward", episode_rew)
def run(policy, env, seed, nsteps=5, nstack=4, total_timesteps=int(80e6), vf_coef=0.5, ent_coef=0.01, max_grad_norm=0.5, lr=7e-4, lrschedule='linear', epsilon=1e-5, alpha=0.99, gamma=0.99, log_interval=100): tf.reset_default_graph() set_global_seeds(seed) nenvs = env.num_envs ob_space = env.observation_space ac_space = env.action_space num_procs = len(env.remotes) model = Model(policy=policy, ob_space=ob_space, ac_space=ac_space, nenvs=nenvs, nsteps=nsteps, nstack=nstack, num_procs=num_procs, ent_coef=ent_coef, vf_coef=vf_coef, max_grad_norm=max_grad_norm, lr=lr, alpha=alpha, epsilon=epsilon, total_timesteps=total_timesteps, lrschedule=lrschedule) model.load('./model/a2c/model.h5') runner = Runner(env, model, nsteps=nsteps, nstack=nstack, gamma=gamma) while True: runner.run()
class ShootEnv(Env): def __init__(self): self.game = DoomGame() self.game.load_config('O:\\Doom\\scenarios\\cig_flat2.cfg') self.game.add_game_args( "-host 1 -deathmatch +timelimit 1.0 " "+sv_forcerespawn 1 +sv_noautoaim 1 +sv_respawnprotect 1 +sv_spawnfarthest 1 +sv_nocrouch 1 " "+viz_respawn_delay 0") self.game.set_mode(Mode.PLAYER) self.game.set_labels_buffer_enabled(True) self.game.set_depth_buffer_enabled(True) self.game.set_screen_resolution(ScreenResolution.RES_320X240) self.game.add_available_game_variable(GameVariable.FRAGCOUNT) #define navigation env class NavigatorSubEnv(Env): def __init__(self, game): self.action_space = Discrete(3) self.observation_space = Box(low=0, high=255, shape=(84, 84, 3), dtype=np.uint8) self._game = game def step(self, action): #-1 means it doesn't really controls the game if action > -1: one_hot_action = [[1, 0, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0], [0, 0, 0, 0, 1, 0]] self._game.make_action(one_hot_action[action], 4) if self._game.is_episode_finished(): self._game.new_episode() if self._game.is_player_dead(): self._game.respawn_player() obs = get_observation(self._game.get_state()) return get_observation( self._game.get_state(), real_frame=True), 0, check_enemy_enter(obs), None def seed(self, seed=None): pass def reset(self): return get_observation(self._game.get_state(), real_frame=True) def render(self, mode='human'): pass self.navigator = VecFrameStack( VecEnvAdapter([NavigatorSubEnv(self.game)]), 4) #define navigation network self.navigation_policy = Model(CnnPolicy, self.navigator.observation_space, self.navigator.action_space, nenvs=1, nsteps=1) self.navigation_policy.load( 'O:\\Doom\\baselinemodel\\navigate_real2.dat') self.action_space = Discrete(3) #turn L, turn R, fire self.observation_space = Box(low=0, high=255, shape=(84, 84, 3), dtype=np.uint8) self.available_actions = [[1, 0, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0], [0, 0, 0, 0, 0, 1]] def seed(self, seed=None): self.game.set_seed(seed) self.game.init() self.game.send_game_command("removebots") for i in range(8): self.game.send_game_command("addbot") def reset(self): obs_for_navigator = self.navigator.reset() while True: actions, _, _, _ = self.navigation_policy.step(obs_for_navigator) obs_for_navigator, _, navi_done, _ = self.navigator.step(actions) if navi_done: break obs = get_observation(self.game.get_state()) assert check_enemy_enter(obs) return get_observation(self.game.get_state(), real_frame=True) def step(self, action): old_fragcount = self.game.get_game_variable(GameVariable.FRAGCOUNT) self.game.make_action(self.available_actions[action], 4) new_fragcount = self.game.get_game_variable(GameVariable.FRAGCOUNT) rew = new_fragcount - old_fragcount done = False if self.game.is_episode_finished(): done = True self.game.new_episode() self.game.send_game_command("removebots") for i in range(8): self.game.send_game_command("addbot") if self.game.is_player_dead(): self.game.respawn_player() done = True if action == 2: # fire rew -= 0.05 state = self.game.get_state() obs = get_observation(state) if check_enemy_enter(obs): rew += 0.01 if check_enemy_leave(obs): done = True return get_observation(state, real_frame=True), rew, done, None
def test(env_id, policy_name, seed, nstack=1, numAgents=2): iters = 100 rwd = [] percent_exp = [] env = gym.make(env_id) env.seed(seed) print("logger dir: ", logger.get_dir()) env = bench.Monitor(env, logger.get_dir() and os.path.join(logger.get_dir())) if env_id == 'Pendulum-v0': if continuous_actions: env.action_space.n = env.action_space.shape[0] else: env.action_space.n = 10 gym.logger.setLevel(logging.WARN) # img_shape = (84, 84, 3) img_shape = (84, 84, 3) ob_space = spaces.Box(low=0, high=255, shape=img_shape) ac_space = env.action_space # def get_img(env): # ax, img = env.get_img() # return ax, img # def process_img(img): # img = rgb2grey(copy.deepcopy(img)) # img = resize(img, img_shape) # return img policy_fn = policy_fn_name(policy_name) nsteps = 5 total_timesteps = int(80e6) vf_coef = 0.5 ent_coef = 0.01 max_grad_norm = 0.5 lr = 7e-4 lrschedule = 'linear' epsilon = 1e-5 alpha = 0.99 continuous_actions = False debug = False if numAgents == 1: model = Model(policy=policy_fn, ob_space=ob_space, ac_space=ac_space, nenvs=1, nsteps=nsteps, nstack=nstack, num_procs=1, ent_coef=ent_coef, vf_coef=vf_coef, max_grad_norm=max_grad_norm, lr=lr, alpha=alpha, epsilon=epsilon, total_timesteps=total_timesteps, lrschedule=lrschedule, continuous_actions=continuous_actions, debug=debug) m_name = 'test_model_Mar7_1mil.pkl' model.load(m_name) else: model = [] for i in range(numAgents): model.append( Model(policy=policy_fn, ob_space=ob_space, ac_space=ac_space, nenvs=1, nsteps=nsteps, nstack=nstack, num_procs=1, ent_coef=ent_coef, vf_coef=vf_coef, max_grad_norm=max_grad_norm, lr=lr, alpha=alpha, epsilon=epsilon, total_timesteps=total_timesteps, lrschedule=lrschedule, continuous_actions=continuous_actions, debug=debug, itr=i)) for i in range(numAgents): m_name = 'test_model_' + str(i) + '_300k.pkl' # + '100k.pkl' model[i].load(m_name) print('---------------------------------------------') print("Successfully Loaded: ", m_name) print('---------------------------------------------') env.env, img = env.reset() rwd = [[], []] percent_exp = [[], []] for i in range(1, iters + 1): if i % 10 == 0: for j in range(numAgents): print('-----------------------------------') print('Agent ' + str(j)) print('Iteration: ', i) avg_rwd = sum(rwd[j]) / i avg_pct_exp = sum(percent_exp[j]) / i med_pct_exp = statistics.median(percent_exp[j]) print('Average Reward: ', avg_rwd) print('Average Percent Explored: ', avg_pct_exp, '%') print('Median Percent Explored: ', med_pct_exp) print('-----------------------------------') # ax, img = get_img(env) img_hist = [] frames_dir = [] for j in range(numAgents): frames_dir.append('exp_frames' + str(j * 100 + i + 200)) if os.path.exists(frames_dir[j]): # raise ValueError('Frames directory already exists.') shutil.rmtree(frames_dir[j]) os.makedirs(frames_dir[j]) img_hist.append(deque([img[j] for _ in range(4)], maxlen=nstack)) action = 0 total_rewards = [0, 0] nstack = 1 for tidx in range(1000): # if tidx % nstack == 0: for j in range(numAgents): if tidx > 0: input_imgs = np.expand_dims( np.squeeze(np.stack(img_hist, -1)), 0) # print(np.shape(input_imgs)) # plt.imshow(input_imgs[0, :, :, 0]) # plt.imshow(input_imgs[0, :, :, 1]) # plt.draw() # plt.pause(0.000001) if input_imgs.shape == (1, 84, 84, 3): actions, values, states = model[j].step_model.step( input_imgs) else: actions, values, states = model[j].step_model.step( input_imgs[:, :, :, :, 0]) # actions, values, states = model.step_model.step(input_imgs) action = actions[0] value = values[0] # print('Value: ', value, ' Action: ', action) img, reward, done, _ = env.step(action, j) total_rewards[j] += reward # img = get_img(env) img_hist[j].append(img[j]) imsave( os.path.join(frames_dir[j], 'frame_{:04d}.png'.format(tidx)), resize(img[j], (img_shape[0], img_shape[1], 3))) # print(tidx, '\tAction: ', action, '\tValues: ', value, '\tRewards: ', reward, '\tTotal rewards: ', total_rewards)#, flush=True) if done: # print('Faultered at tidx: ', tidx) for j in range(numAgents): rwd[j].append(total_rewards[j]) percent_exp[j].append(env.env.percent_explored[j]) # env.env, img = env.reset() break for i in range(numAgents): print('-----------------------------------') print('Agent ' + str(i)) print('Iteration: ', iters) avg_rwd = sum(rwd[i]) / iters avg_pct_exp = sum(percent_exp[i]) / iters med_pct_exp = statistics.median(percent_exp[i]) print('Average Reward: ', avg_rwd) print('Average Percent Explored: ', avg_pct_exp, '%') print('Median Percent Explored: ', med_pct_exp) print('-----------------------------------')
def test(env_id, policy_name, seed, nstack=1, numAgents=2, benchmark=False): iters = 100 rwd = [] percent_exp = [] env = EnvVec([ make_env(env_id, benchmark=benchmark, rank=idx, seed=seed) for idx in range(1) ], particleEnv=True, test=True) # print(env_id) # print("logger dir: ", logger.get_dir()) # env = bench.Monitor(env, logger.get_dir() and os.path.join(logger.get_dir())) if env_id == 'Pendulum-v0': if continuous_actions: env.action_space.n = env.action_space.shape[0] else: env.action_space.n = 10 gym.logger.setLevel(logging.WARN) ob_space = env.observation_space ac_space = env.action_space # def get_img(env): # ax, img = env.get_img() # return ax, img # def process_img(img): # img = rgb2grey(copy.deepcopy(img)) # img = resize(img, img_shape) # return img policy_fn = policy_fn_name(policy_name) nsteps = 5 total_timesteps = int(80e6) vf_coef = 0.9 ent_coef = 0.01 max_grad_norm = 0.5 lr = 7e-4 lrschedule = 'linear' epsilon = 1e-5 alpha = 0.99 continuous_actions = False debug = False if numAgents == 1: model = Model(policy=policy_fn, ob_space=ob_space, ac_space=ac_space, nenvs=1, nsteps=nsteps, nstack=nstack, num_procs=1, ent_coef=ent_coef, vf_coef=vf_coef, max_grad_norm=max_grad_norm, lr=lr, alpha=alpha, epsilon=epsilon, total_timesteps=total_timesteps, lrschedule=lrschedule, continuous_actions=continuous_actions, debug=debug) m_name = 'test_model_Mar7_1mil.pkl' model.load(m_name) else: model = [] for i in range(numAgents): model.append( Model(policy=policy_fn, ob_space=ob_space, ac_space=ac_space, nenvs=1, nsteps=nsteps, nstack=nstack, num_procs=1, ent_coef=ent_coef, vf_coef=vf_coef, max_grad_norm=max_grad_norm, lr=lr, alpha=alpha, epsilon=epsilon, total_timesteps=total_timesteps, lrschedule=lrschedule, continuous_actions=continuous_actions, debug=debug, itr=i, particleEnv=True)) for i in range(numAgents): m_name = 'partEnv_model_' + str(i) + '.pkl' model[i].load(m_name) print('---------------------------------------------') print("Successfully Loaded: ", m_name) print('---------------------------------------------') obs = env.reset() states = [[], []] dones = [False, False] rwd = [[], []] percent_exp = [[], []] for i in range(1, iters + 1): if i % 1 == 0: for j in range(numAgents): print('-----------------------------------') print('Agent ' + str(j)) print('Iteration: ', i) avg_rwd = sum(rwd[j]) / i # avg_pct_exp = sum(percent_exp[j])/i # med_pct_exp = statistics.median(percent_exp[j]) print('Average Reward: ', avg_rwd) # print('Average Percent Explored: ', avg_pct_exp, '%') # print('Median Percent Explored: ', med_pct_exp) print('-----------------------------------') actions = [[], []] values = [[], []] total_rewards = [[0], [0]] nstack = 1 for tidx in range(1000): # if tidx % nstack == 0: for j in range(numAgents): # if tidx > 0: # input_imgs = np.expand_dims(np.squeeze(np.stack(img_hist, -1)), 0) # print(np.shape(input_imgs)) # plt.imshow(input_imgs[0, :, :, 0]) # plt.imshow(input_imgs[0, :, :, 1]) # plt.draw() # plt.pause(0.000001) # print(obs[:, j]) # print(states[j]) # print(dones) # actions[j], values[j], states[j] = model[j].step(obs[:, j].reshape(1, 21), states[j], dones[j]) ob_shape = np.asarray([ env.observation_space[i].shape for i in range(env.n) ]).flatten() print(ob_shape) actions[j], values[j], states[j] = model[j].step( obs[:, j].reshape(1, ob_shape[1]), states[j], dones[j]) # action = actions[0] # value = values[0] obs, rewards, dones, _ = env.step(actions) dones = dones.flatten() total_rewards += rewards # wrong? print(total_rewards) # print(dones) # img = get_img(env) # obs_hist[j].append(img[j]) # imsave(os.path.join(frames_dir[j], 'frame_{:04d}.png'.format(tidx)), resize(img[j], (img_shape[0], img_shape[1], 3))) # print(tidx, '\tAction: ', action, '\tValues: ', value, '\tRewards: ', reward, '\tTotal rewards: ', total_rewards)#, flush=True) if True in dones: # print('Faultered at tidx: ', tidx) for j in range(numAgents): rwd[j].append(total_rewards[j]) # percent_exp[j].append(env.env.percent_explored[j]) obs = env.reset() break for i in range(numAgents): print('-----------------------------------') print('Agent ' + str(i)) print('Iteration: ', iters) avg_rwd = sum(rwd[i]) / iters # avg_pct_exp = sum(percent_exp[i])/iters # med_pct_exp = statistics.median(percent_exp[i]) print('Average Reward: ', avg_rwd) # print('Average Percent Explored: ', avg_pct_exp, '%') # print('Median Percent Explored: ', med_pct_exp) print('-----------------------------------')
def main(visualize=False): session = tf_util.make_session() env_model = EnvNetwork(action_space_size=6, nbatch=num_env * singlestep, K=K, nsteps=singlestep, reuse=False, session=session) session.run(tf.global_variables_initializer()) env_model.restore() env = VecFrameStack(make_doom_env(num_env, seed, 'mixed'), 4) navi_model = Model(policy=CnnPolicy, ob_space=env.observation_space, ac_space=Discrete(3), nenvs=num_env, nsteps=nsteps, ent_coef=0.01, vf_coef=0.5, max_grad_norm=0.5, lr=7e-4, alpha=0.99, epsilon=1e-5, total_timesteps=total_timesteps, lrschedule='linear', model_name='navi') navi_model.load("O:\\Doom\\baselinemodel\\navigate_flat2.dat") fire_model = Model(policy=CnnPolicy, ob_space=env.observation_space, ac_space=Discrete(3), nenvs=num_env, nsteps=nsteps, ent_coef=0.01, vf_coef=0.5, max_grad_norm=0.5, lr=7e-4, alpha=0.99, epsilon=1e-5, total_timesteps=total_timesteps, lrschedule='linear', model_name='fire') fire_model.load("O:\\Doom\\baselinemodel\\fire_flat2.dat") policy_model = MixedModel(navi_model, fire_model, check_enemy_leave, check_enemy_enter, [0, 1, 4], [0, 1, 5]) runner = Runner(env, policy_model, nsteps=nsteps, gamma=0.99) nh, nw, nc = env.observation_space.shape while True: total_loss = 0 for _ in tqdm(range(save_freq)): obs1, _, _, mask1, actions1, _ = runner.run() obs1 = np.reshape(obs1, [num_env, nsteps, nh, nw, nc]) obs1 = obs1[:, :, :, :, -1:] actions1 = np.reshape(actions1, [num_env, nsteps]) mask1 = np.reshape(mask1, [num_env, nsteps]) hidden_states = env_model.initial_state for s in range(0, nsteps - K - singlestep, singlestep): input_frames = obs1[:, s:s + singlestep, :, :, :] // norm_factor input_frames = np.reshape(input_frames, [num_env * singlestep, nh, nw]) input_frames = np.eye(9)[input_frames] actions, masks, expected_observations = [], [], [] for t in range(K): expected_observation = obs1[:, s + t + 1:s + singlestep + t + 1, :, :, :] expected_observation = np.reshape( expected_observation, [num_env * singlestep, nh, nw, 1]) expected_observations.append(expected_observation) action = actions1[:, s + t:s + singlestep + t] action = np.reshape(action, [num_env * singlestep]) actions.append(action) mask = mask1[:, s + t:s + singlestep + t] mask = np.reshape(mask, [num_env * singlestep]) masks.append(mask) if s > 0: loss, prediction, hidden_states = env_model.train_and_predict( input_frames, actions, masks, expected_observations, hidden_states) total_loss += loss else: # warm up prediction, hidden_states = env_model.predict( input_frames, actions, masks, hidden_states) if visualize and s == 3 * singlestep: for batch_idx in range(num_env * singlestep): expected_t = expected_observations[0] if np.sum(expected_t[batch_idx, :, :, :] > 0.0): input_frame = input_frames[batch_idx, :, :, :] cv2.imshow('input', input_frame) for i in range(K): time_t_expectation = expected_observations[i] exp_obs = time_t_expectation[ batch_idx, :, :, :] cv2.imshow('expected for t+{}'.format(i + 1), exp_obs) for i in range(K): time_t_prediction = prediction[i] cv2.imshow( 'prediction for t+{}'.format(i + 1), time_t_prediction[batch_idx, :, :, 7]) cv2.waitKey(0) print("avg_loss = {}".format(total_loss / K / save_freq / valid_batch_size)) env_model.save()