def __init__(self, config):
self.config = config
self.sample_data_queue = queue.Queue(
maxsize=config['sample_queue_max_size'])
#=========== Create Agent ==========
env = gym.make(config['env_name'])
env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')
obs_shape = env.observation_space.shape
act_dim = env.action_space.n
model = AtariModel(act_dim)
algorithm = parl.algorithms.IMPALA(
model,
sample_batch_steps=self.config['sample_batch_steps'],
gamma=self.config['gamma'],
vf_loss_coeff=self.config['vf_loss_coeff'],
clip_rho_threshold=self.config['clip_rho_threshold'],
clip_pg_rho_threshold=self.config['clip_pg_rho_threshold'])
self.agent = AtariAgent(algorithm, obs_shape, act_dim,
self.learn_data_provider)
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_TO_USE]` .'
self.cache_params = self.agent.get_weights()
self.params_lock = threading.Lock()
self.params_updated = False
self.cache_params_sent_cnt = 0
self.total_params_sync = 0
#========== Learner ==========
self.lr, self.entropy_coeff = None, None
self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.kl_stat = WindowStat(100)
self.learn_time_stat = TimeStat(100)
self.start_time = None
self.learn_thread = threading.Thread(target=self.run_learn)
self.learn_thread.setDaemon(True)
self.learn_thread.start()
#========== Remote Actor ===========
self.remote_count = 0
self.batch_buffer = []
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.create_actors()
def __init__(self, config):
self.config = config
self.sample_data_queue = queue.Queue(
maxsize=config['sample_queue_max_size'])
#=========== Create Agent ==========
env = IntraBuildingEnv("config.ini")
self._mansion_attr = env._mansion.attribute
self._obs_dim = obs_dim(self._mansion_attr)
self._act_dim = act_dim(self._mansion_attr)
self.config['obs_shape'] = self._obs_dim
self.config['act_dim'] = self._act_dim
model = RLDispatcherModel(self._act_dim)
algorithm = IMPALA(model, hyperparas=config)
self.agent = ElevatorAgent(algorithm, config, self.learn_data_provider)
self.cache_params = self.agent.get_params()
self.params_lock = threading.Lock()
self.params_updated = False
self.cache_params_sent_cnt = 0
self.total_params_sync = 0
#========== Learner ==========
self.lr, self.entropy_coeff = None, None
self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.kl_stat = WindowStat(100)
self.learn_time_stat = TimeStat(100)
self.start_time = None
self.learn_thread = threading.Thread(target=self.run_learn)
self.learn_thread.setDaemon(True)
self.learn_thread.start()
#========== Remote Actor ===========
self.remote_count = 0
self.batch_buffer = []
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.remote_manager_thread = threading.Thread(
target=self.run_remote_manager)
self.remote_manager_thread.setDaemon(True)
self.remote_manager_thread.start()
self.csv_logger = CSVLogger(
os.path.join(logger.get_dir(), 'result.csv'))
from utils import Summary
self.summary = Summary('./output')
def __init__(self, algorithm, config, obs_dim):
"""
Args:
algorithm (`parl.Algorithm`): 强学习算法
config (dict): 配置文件参数
"""
self.obs_dim = obs_dim
super(Agent, self).__init__(algorithm)
# 学习率衰减
self.lr_scheduler = LinearDecayScheduler(config['start_lr'], config['max_sample_steps'])
self.entropy_coeff_scheduler = PiecewiseScheduler(config['entropy_coeff_scheduler'])
def __init__(self, model, config):
assert isinstance(config['vf_loss_coeff'], (int, float))
self.model = model
self.vf_loss_coeff = config['vf_loss_coeff']
self.optimizer = optim.Adam(self.model.parameters(),
lr=config['learning_rate'])
self.config = config
self.lr_scheduler = LinearDecayScheduler(config['start_lr'],
config['max_sample_steps'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
def __init__(self, algorithm, config):
"""
Args:
algorithm (`parl.Algorithm`): algorithm to be used in this agent.
config (dict): config file describing the training hyper-parameters(see a2c_config.py)
"""
self.obs_shape = config['obs_shape']
super(AtariAgent, self).__init__(algorithm)
self.lr_scheduler = LinearDecayScheduler(config['start_lr'],
config['max_sample_steps'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
def __init__(self, config):
self.config = config
self.sample_data_queue = queue.Queue()
self.batch_buffer = defaultdict(list)
#=========== Create Agent ==========
env = gym.make(config['env_name'])
env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')
obs_shape = env.observation_space.shape
act_dim = env.action_space.n
self.config['obs_shape'] = obs_shape
self.config['act_dim'] = act_dim
model = AtariModel(act_dim)
algorithm = parl.algorithms.A3C(model,
vf_loss_coeff=config['vf_loss_coeff'])
self.agent = AtariAgent(
algorithm,
obs_shape=self.config['obs_shape'],
predict_thread_num=self.config['predict_thread_num'],
learn_data_provider=self.learn_data_provider)
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_YOU_WANT_TO_USE]` .'
else:
cpu_num = os.environ.get('CPU_NUM')
assert cpu_num is not None and cpu_num == '1', 'Only support training in single CPU,\
Please set environment variable: `export CPU_NUM=1`.'
#========== Learner ==========
self.lr, self.entropy_coeff = None, None
self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.learn_time_stat = TimeStat(100)
self.start_time = None
# learn thread
self.learn_thread = threading.Thread(target=self.run_learn)
self.learn_thread.setDaemon(True)
self.learn_thread.start()
self.predict_input_queue = queue.Queue()
# predict thread
self.predict_threads = []
for i in six.moves.range(self.config['predict_thread_num']):
predict_thread = threading.Thread(target=self.run_predict,
args=(i, ))
predict_thread.setDaemon(True)
predict_thread.start()
self.predict_threads.append(predict_thread)
#========== Remote Simulator ===========
self.remote_count = 0
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.create_actors()