def main(): env = get_player( args.rom, image_size=IMAGE_SIZE, train=True, frame_skip=FRAME_SKIP) file_path = "memory.npz" rpm = ReplayMemory( MEMORY_SIZE, IMAGE_SIZE, CONTEXT_LEN, load_file=True, # load replay memory data from file file_path=file_path) act_dim = env.action_space.n model = AtariModel(act_dim) algorithm = DQN( model, act_dim=act_dim, gamma=GAMMA, lr=LEARNING_RATE * gpu_num) agent = AtariAgent( algorithm, act_dim=act_dim, total_step=args.train_total_steps) if os.path.isfile('./model.ckpt'): logger.info("load model from file") agent.restore('./model.ckpt') if args.train: logger.info("train with memory data") run_train_step(agent, rpm) logger.info("finish training. Save the model.") agent.save('./model.ckpt') else: logger.info("collect experience") collect_exp(env, rpm, agent) rpm.save_memory() logger.info("finish collecting, save successfully")
def __init__(self, config): self.config = config self.sample_data_queue = queue.Queue( maxsize=config['sample_queue_max_size']) #=========== Create Agent ========== env = gym.make(config['env_name']) env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW') obs_shape = env.observation_space.shape act_dim = env.action_space.n model = AtariModel(act_dim) algorithm = parl.algorithms.IMPALA( model, sample_batch_steps=self.config['sample_batch_steps'], gamma=self.config['gamma'], vf_loss_coeff=self.config['vf_loss_coeff'], clip_rho_threshold=self.config['clip_rho_threshold'], clip_pg_rho_threshold=self.config['clip_pg_rho_threshold']) self.agent = AtariAgent(algorithm, obs_shape, act_dim, self.learn_data_provider) if machine_info.is_gpu_available(): assert get_gpu_count() == 1, 'Only support training in single GPU,\ Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_TO_USE]` .' self.cache_params = self.agent.get_weights() self.params_lock = threading.Lock() self.params_updated = False self.cache_params_sent_cnt = 0 self.total_params_sync = 0 #========== Learner ========== self.lr, self.entropy_coeff = None, None self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler']) self.entropy_coeff_scheduler = PiecewiseScheduler( config['entropy_coeff_scheduler']) self.total_loss_stat = WindowStat(100) self.pi_loss_stat = WindowStat(100) self.vf_loss_stat = WindowStat(100) self.entropy_stat = WindowStat(100) self.kl_stat = WindowStat(100) self.learn_time_stat = TimeStat(100) self.start_time = None self.learn_thread = threading.Thread(target=self.run_learn) self.learn_thread.setDaemon(True) self.learn_thread.start() #========== Remote Actor =========== self.remote_count = 0 self.batch_buffer = [] self.remote_metrics_queue = queue.Queue() self.sample_total_steps = 0 self.create_actors()
def main(): env = get_player(args.rom, image_size=IMAGE_SIZE, train=True, frame_skip=FRAME_SKIP) test_env = get_player(args.rom, image_size=IMAGE_SIZE, frame_skip=FRAME_SKIP, context_len=CONTEXT_LEN) rpm = ReplayMemory(MEMORY_SIZE, IMAGE_SIZE, CONTEXT_LEN) act_dim = env.action_space.n model = AtariModel(act_dim, args.algo) if args.algo == 'DDQN': algorithm = parl.algorithms.DDQN(model, act_dim=act_dim, gamma=GAMMA) elif args.algo in ['DQN', 'Dueling']: algorithm = parl.algorithms.DQN(model, act_dim=act_dim, gamma=GAMMA) agent = AtariAgent(algorithm, act_dim=act_dim, start_lr=LEARNING_RATE, total_step=args.train_total_steps, update_freq=UPDATE_FREQ) with tqdm(total=MEMORY_WARMUP_SIZE, desc='[Replay Memory Warm Up]') as pbar: while rpm.size() < MEMORY_WARMUP_SIZE: total_reward, steps, _ = run_train_episode(env, agent, rpm) pbar.update(steps) # train test_flag = 0 pbar = tqdm(total=args.train_total_steps) total_steps = 0 max_reward = None while total_steps < args.train_total_steps: # start epoch total_reward, steps, loss = run_train_episode(env, agent, rpm) total_steps += steps pbar.set_description('[train]exploration:{}'.format(agent.exploration)) summary.add_scalar('dqn/score', total_reward, total_steps) summary.add_scalar('dqn/loss', loss, total_steps) # mean of total loss summary.add_scalar('dqn/exploration', agent.exploration, total_steps) pbar.update(steps) if total_steps // args.test_every_steps >= test_flag: while total_steps // args.test_every_steps >= test_flag: test_flag += 1 pbar.write("testing") eval_rewards = [] for _ in tqdm(range(3), desc='eval agent'): eval_reward = run_evaluate_episode(test_env, agent) eval_rewards.append(eval_reward) logger.info( "eval_agent done, (steps, eval_reward): ({}, {})".format( total_steps, np.mean(eval_rewards))) eval_test = np.mean(eval_rewards) summary.add_scalar('dqn/eval', eval_test, total_steps) pbar.close()
def __init__(self, config): self.config = config #=========== Create Agent ========== env = gym.make(config['env_name']) env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW') obs_shape = env.observation_space.shape act_dim = env.action_space.n self.config['obs_shape'] = obs_shape self.config['act_dim'] = act_dim model = AtariModel(act_dim) algorithm = parl.algorithms.A3C(model, vf_loss_coeff=config['vf_loss_coeff']) self.agent = AtariAgent(algorithm, config) if machine_info.is_gpu_available(): assert get_gpu_count() == 1, 'Only support training in single GPU,\ Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_TO_USE]` .' #========== Learner ========== self.total_loss_stat = WindowStat(100) self.pi_loss_stat = WindowStat(100) self.vf_loss_stat = WindowStat(100) self.entropy_stat = WindowStat(100) self.lr = None self.entropy_coeff = None self.learn_time_stat = TimeStat(100) self.start_time = None #========== Remote Actor =========== self.remote_count = 0 self.sample_data_queue = queue.Queue() self.remote_metrics_queue = queue.Queue() self.sample_total_steps = 0 self.params_queues = [] self.create_actors()
def __init__(self, config): self.config = config self.envs = [] for _ in range(config['env_num']): env = gym.make(config['env_name']) env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW') self.envs.append(env) self.vector_env = VectorEnv(self.envs) self.obs_batch = self.vector_env.reset() obs_shape = env.observation_space.shape act_dim = env.action_space.n self.config['obs_shape'] = obs_shape self.config['act_dim'] = act_dim model = AtariModel(act_dim) algorithm = parl.algorithms.A3C(model, vf_loss_coeff=config['vf_loss_coeff']) self.agent = AtariAgent(algorithm, config)
def __init__(self, config): self.config = config self.envs = [] for _ in range(config['env_num']): env = gym.make(config['env_name']) env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW') self.envs.append(env) self.vector_env = VectorEnv(self.envs) self.obs_batch = self.vector_env.reset() obs_shape = env.observation_space.shape act_dim = env.action_space.n model = AtariModel(act_dim) algorithm = parl.algorithms.IMPALA( model, sample_batch_steps=self.config['sample_batch_steps'], gamma=self.config['gamma'], vf_loss_coeff=self.config['vf_loss_coeff'], clip_rho_threshold=self.config['clip_rho_threshold'], clip_pg_rho_threshold=self.config['clip_pg_rho_threshold']) self.agent = AtariAgent(algorithm, obs_shape, act_dim)
def main(): # Prepare environments env = get_player( args.rom, image_size=IMAGE_SIZE, train=True, frame_skip=FRAME_SKIP) test_env = get_player( args.rom, image_size=IMAGE_SIZE, frame_skip=FRAME_SKIP, context_len=CONTEXT_LEN) # Init Prioritized Replay Memory per = ProportionalPER(alpha=0.6, seg_num=args.batch_size, size=MEMORY_SIZE) # Prepare PARL agent act_dim = env.action_space.n model = AtariModel(act_dim) if args.alg == 'ddqn': algorithm = PrioritizedDoubleDQN( model, act_dim=act_dim, gamma=GAMMA, lr=LEARNING_RATE) elif args.alg == 'dqn': algorithm = PrioritizedDQN( model, act_dim=act_dim, gamma=GAMMA, lr=LEARNING_RATE) agent = AtariAgent(algorithm, act_dim=act_dim, update_freq=UPDATE_FREQ) # Replay memory warmup total_step = 0 with tqdm(total=MEMORY_SIZE, desc='[Replay Memory Warm Up]') as pbar: mem = [] while total_step < MEMORY_WARMUP_SIZE: total_reward, steps, _ = run_episode( env, agent, per, mem=mem, warmup=True) total_step += steps pbar.update(steps) per.elements.from_list(mem[:int(MEMORY_WARMUP_SIZE)]) env_name = args.rom.split('/')[-1].split('.')[0] test_flag = 0 total_steps = 0 pbar = tqdm(total=args.train_total_steps) while total_steps < args.train_total_steps: # start epoch total_reward, steps, loss = run_episode(env, agent, per, train=True) total_steps += steps pbar.set_description('[train]exploration:{}'.format(agent.exploration)) summary.add_scalar('{}/score'.format(env_name), total_reward, total_steps) summary.add_scalar('{}/loss'.format(env_name), loss, total_steps) # mean of total loss summary.add_scalar('{}/exploration'.format(env_name), agent.exploration, total_steps) pbar.update(steps) if total_steps // args.test_every_steps >= test_flag: while total_steps // args.test_every_steps >= test_flag: test_flag += 1 pbar.write("testing") test_rewards = [] for _ in tqdm(range(3), desc='eval agent'): eval_reward = run_evaluate_episode(test_env, agent) test_rewards.append(eval_reward) eval_reward = np.mean(test_rewards) logger.info( "eval_agent done, (steps, eval_reward): ({}, {})".format( total_steps, eval_reward)) summary.add_scalar('{}/eval'.format(env_name), eval_reward, total_steps) pbar.close()
def __init__(self, config): self.config = config self.sample_data_queue = queue.Queue() self.batch_buffer = defaultdict(list) #=========== Create Agent ========== env = gym.make(config['env_name']) env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW') obs_shape = env.observation_space.shape act_dim = env.action_space.n self.config['obs_shape'] = obs_shape self.config['act_dim'] = act_dim model = AtariModel(act_dim) algorithm = parl.algorithms.A3C(model, vf_loss_coeff=config['vf_loss_coeff']) self.agent = AtariAgent( algorithm, obs_shape=self.config['obs_shape'], predict_thread_num=self.config['predict_thread_num'], learn_data_provider=self.learn_data_provider) if machine_info.is_gpu_available(): assert get_gpu_count() == 1, 'Only support training in single GPU,\ Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_YOU_WANT_TO_USE]` .' else: cpu_num = os.environ.get('CPU_NUM') assert cpu_num is not None and cpu_num == '1', 'Only support training in single CPU,\ Please set environment variable: `export CPU_NUM=1`.' #========== Learner ========== self.lr, self.entropy_coeff = None, None self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler']) self.entropy_coeff_scheduler = PiecewiseScheduler( config['entropy_coeff_scheduler']) self.total_loss_stat = WindowStat(100) self.pi_loss_stat = WindowStat(100) self.vf_loss_stat = WindowStat(100) self.entropy_stat = WindowStat(100) self.learn_time_stat = TimeStat(100) self.start_time = None # learn thread self.learn_thread = threading.Thread(target=self.run_learn) self.learn_thread.setDaemon(True) self.learn_thread.start() self.predict_input_queue = queue.Queue() # predict thread self.predict_threads = [] for i in six.moves.range(self.config['predict_thread_num']): predict_thread = threading.Thread(target=self.run_predict, args=(i, )) predict_thread.setDaemon(True) predict_thread.start() self.predict_threads.append(predict_thread) #========== Remote Simulator =========== self.remote_count = 0 self.remote_metrics_queue = queue.Queue() self.sample_total_steps = 0 self.create_actors()
break return total_reward if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--game_name', default='Phoenix-v0') test_env = get_player('Phoenix-v0', image_size=IMAGE_SIZE, context_len=CONTEXT_LEN) save_path = './dqn_model.ckpt' act_dim = test_env.action_space.n model = AtariModel(act_dim) algorithm = parl.algorithms.DQN(model, act_dim=act_dim, gamma=GAMMA) agent = AtariAgent(algorithm, act_dim=act_dim, start_lr=LEARNING_RATE, total_step=test_number, update_freq=UPDATE_FREQ) agent.restore(save_path) eval_rewards = [] flag = 0 while flag < test_number: eval_reward = run_evaluate_episode(test_env, agent)
def main(): # Prepare environments # env = get_player( # args.rom, image_size=IMAGE_SIZE, train=True, frame_skip=FRAME_SKIP) # test_env = get_player( # args.rom, # image_size=IMAGE_SIZE, # frame_skip=FRAME_SKIP, # context_len=CONTEXT_LEN) env = gym.make("pseudoslam:RobotExploration-v0") env = MonitorEnv(env, param={'goal': args.goal, 'obs': args.obs}) # obs = env.reset() # print(obs.shape) # raise NotImplementedError # Init Prioritized Replay Memory per = ProportionalPER(alpha=0.6, seg_num=args.batch_size, size=MEMORY_SIZE) suffix = args.suffix + "_Rp{}_Goal{}_Obs{}".format(args.Rp, args.goal, args.obs) logdir = os.path.join(args.logdir, suffix) if not os.path.exists(logdir): os.mkdir(logdir) logger.set_dir(logdir) modeldir = os.path.join(args.modeldir, suffix) if not os.path.exists(modeldir): os.mkdir(modeldir) # Prepare PARL agent act_dim = env.action_space.n model = AtariModel(act_dim) if args.alg == 'ddqn': algorithm = PrioritizedDoubleDQN(model, act_dim=act_dim, gamma=GAMMA, lr=LEARNING_RATE) elif args.alg == 'dqn': algorithm = PrioritizedDQN(model, act_dim=act_dim, gamma=GAMMA, lr=LEARNING_RATE) agent = AtariAgent(algorithm, act_dim=act_dim, update_freq=UPDATE_FREQ) if os.path.exists(args.load): agent.restore(args.load) # Replay memory warmup total_step = 0 with tqdm(total=MEMORY_SIZE, desc='[Replay Memory Warm Up]') as pbar: mem = [] while total_step < MEMORY_WARMUP_SIZE: total_reward, steps, _, _ = run_episode(env, agent, per, mem=mem, warmup=True) total_step += steps pbar.update(steps) per.elements.from_list(mem[:int(MEMORY_WARMUP_SIZE)]) # env_name = args.rom.split('/')[-1].split('.')[0] test_flag = 0 total_steps = 0 pbar = tqdm(total=args.train_total_steps) save_steps = 0 while total_steps < args.train_total_steps: # start epoch total_reward, steps, loss, info = run_episode(env, agent, per, train=True) total_steps += steps save_steps += steps pbar.set_description('[train]exploration:{}'.format(agent.exploration)) summary.add_scalar('train/score', total_reward, total_steps) summary.add_scalar('train/loss', loss, total_steps) # mean of total loss summary.add_scalar('train/exploration', agent.exploration, total_steps) summary.add_scalar('train/steps', steps, total_steps) for key in info.keys(): summary.add_scalar('train/' + key, info[key], total_steps) pbar.update(steps) if total_steps // args.test_every_steps >= test_flag: print('start test!') while total_steps // args.test_every_steps >= test_flag: test_flag += 1 pbar.write("testing") test_rewards = [] for _ in tqdm(range(3), desc='eval agent'): eval_reward = run_evaluate_episode(env, agent) test_rewards.append(eval_reward) eval_reward = np.mean(test_rewards) logger.info( "eval_agent done, (steps, eval_reward): ({}, {})".format( total_steps, eval_reward)) summary.add_scalar('eval/reward', eval_reward, total_steps) if save_steps >= 100000: modeldir_ = os.path.join(modeldir, 'itr_{}'.format(total_steps)) if not os.path.exists(modeldir_): os.mkdir(modeldir_) print('save model!', modeldir_) agent.save(modeldir_) save_steps = 0 pbar.close()