def __init__(self, config): # the cluster may not have gpu os.environ['CUDA_VISIBLE_DEVICES'] = '0' self.actor_cuda = False self.config = config self.envs = [] for _ in range(config['env_num']): env = gym.make(config['env_name']) env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW') self.envs.append(env) self.vector_env = VectorEnv(self.envs) self.obs_batch = self.vector_env.reset() obs_shape = env.observation_space.shape act_dim = env.action_space.n self.config['obs_shape'] = obs_shape self.config['act_dim'] = act_dim model = ActorCritic(act_dim) if self.actor_cuda: model = model.cuda() algorithm = A2C(model, config) self.agent = Agent(algorithm, config)
def __init__(self, config): self.config = config self.sample_data_queue = queue.Queue( maxsize=config['sample_queue_max_size']) #=========== Create Agent ========== env = gym.make(config['env_name']) env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW') obs_shape = env.observation_space.shape act_dim = env.action_space.n model = AtariModel(act_dim) algorithm = parl.algorithms.IMPALA( model, sample_batch_steps=self.config['sample_batch_steps'], gamma=self.config['gamma'], vf_loss_coeff=self.config['vf_loss_coeff'], clip_rho_threshold=self.config['clip_rho_threshold'], clip_pg_rho_threshold=self.config['clip_pg_rho_threshold']) self.agent = AtariAgent(algorithm, obs_shape, act_dim, self.learn_data_provider) if machine_info.is_gpu_available(): assert get_gpu_count() == 1, 'Only support training in single GPU,\ Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_TO_USE]` .' self.cache_params = self.agent.get_weights() self.params_lock = threading.Lock() self.params_updated = False self.cache_params_sent_cnt = 0 self.total_params_sync = 0 #========== Learner ========== self.lr, self.entropy_coeff = None, None self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler']) self.entropy_coeff_scheduler = PiecewiseScheduler( config['entropy_coeff_scheduler']) self.total_loss_stat = WindowStat(100) self.pi_loss_stat = WindowStat(100) self.vf_loss_stat = WindowStat(100) self.entropy_stat = WindowStat(100) self.kl_stat = WindowStat(100) self.learn_time_stat = TimeStat(100) self.start_time = None self.learn_thread = threading.Thread(target=self.run_learn) self.learn_thread.setDaemon(True) self.learn_thread.start() #========== Remote Actor =========== self.remote_count = 0 self.batch_buffer = [] self.remote_metrics_queue = queue.Queue() self.sample_total_steps = 0 self.create_actors()
def __init__(self, config, cuda): self.cuda = cuda self.config = config env = gym.make(config['env_name']) env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW') obs_shape = env.observation_space.shape act_dim = env.action_space.n self.config['obs_shape'] = obs_shape self.config['act_dim'] = act_dim model = ActorCritic(act_dim) if self.cuda: model = model.cuda() algorithm = A2C(model, config) self.agent = Agent(algorithm, config) if machine_info.is_gpu_available(): assert get_gpu_count() == 1, 'Only support training in single GPU,\ Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_YOU_WANT_TO_USE]` .' else: os.environ['CPU_NUM'] = str(1) #========== Learner ========== self.total_loss_stat = WindowStat(100) self.pi_loss_stat = WindowStat(100) self.vf_loss_stat = WindowStat(100) self.entropy_stat = WindowStat(100) self.lr = None self.entropy_coeff = None self.learn_time_stat = TimeStat(100) self.start_time = None #========== Remote Actor =========== self.remote_count = 0 self.sample_total_steps = 0 self.sample_data_queue = queue.Queue() self.remote_metrics_queue = queue.Queue() self.params_queues = [] self.create_actors()
def __init__(self, config): self.config = config self.envs = [] for _ in range(config['env_num']): env = gym.make(config['env_name']) env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW') self.envs.append(env) self.vector_env = VectorEnv(self.envs) self.obs_batch = self.vector_env.reset() obs_shape = env.observation_space.shape act_dim = env.action_space.n self.config['obs_shape'] = obs_shape self.config['act_dim'] = act_dim model = AtariModel(act_dim) algorithm = parl.algorithms.A3C(model, vf_loss_coeff=config['vf_loss_coeff']) self.agent = AtariAgent(algorithm, config)
def __init__(self, config): self.config = config self.envs = [] for _ in range(config['env_num']): env = gym.make(config['env_name']) env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW') self.envs.append(env) self.vector_env = VectorEnv(self.envs) self.obs_batch = self.vector_env.reset() obs_shape = env.observation_space.shape act_dim = env.action_space.n model = AtariModel(act_dim) algorithm = parl.algorithms.IMPALA( model, sample_batch_steps=self.config['sample_batch_steps'], gamma=self.config['gamma'], vf_loss_coeff=self.config['vf_loss_coeff'], clip_rho_threshold=self.config['clip_rho_threshold'], clip_pg_rho_threshold=self.config['clip_pg_rho_threshold']) self.agent = AtariAgent(algorithm, obs_shape, act_dim)
def __init__(self, config): self.config = config self.sample_data_queue = queue.Queue() self.batch_buffer = defaultdict(list) #=========== Create Agent ========== env = gym.make(config['env_name']) env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW') obs_shape = env.observation_space.shape act_dim = env.action_space.n self.config['obs_shape'] = obs_shape self.config['act_dim'] = act_dim model = AtariModel(act_dim) algorithm = parl.algorithms.A3C(model, vf_loss_coeff=config['vf_loss_coeff']) self.agent = AtariAgent( algorithm, obs_shape=self.config['obs_shape'], predict_thread_num=self.config['predict_thread_num'], learn_data_provider=self.learn_data_provider) if machine_info.is_gpu_available(): assert get_gpu_count() == 1, 'Only support training in single GPU,\ Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_YOU_WANT_TO_USE]` .' else: cpu_num = os.environ.get('CPU_NUM') assert cpu_num is not None and cpu_num == '1', 'Only support training in single CPU,\ Please set environment variable: `export CPU_NUM=1`.' #========== Learner ========== self.lr, self.entropy_coeff = None, None self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler']) self.entropy_coeff_scheduler = PiecewiseScheduler( config['entropy_coeff_scheduler']) self.total_loss_stat = WindowStat(100) self.pi_loss_stat = WindowStat(100) self.vf_loss_stat = WindowStat(100) self.entropy_stat = WindowStat(100) self.learn_time_stat = TimeStat(100) self.start_time = None # learn thread self.learn_thread = threading.Thread(target=self.run_learn) self.learn_thread.setDaemon(True) self.learn_thread.start() self.predict_input_queue = queue.Queue() # predict thread self.predict_threads = [] for i in six.moves.range(self.config['predict_thread_num']): predict_thread = threading.Thread(target=self.run_predict, args=(i, )) predict_thread.setDaemon(True) predict_thread.start() self.predict_threads.append(predict_thread) #========== Remote Simulator =========== self.remote_count = 0 self.remote_metrics_queue = queue.Queue() self.sample_total_steps = 0 self.create_actors()
def __init__(self, config): self.config = config env = gym.make(config['env_name']) self.env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')