class Learner(object):
def __init__(self, config):
self.config = config
self.sample_data_queue = queue.Queue()
self.batch_buffer = defaultdict(list)
#=========== Create Agent ==========
env = gym.make(config['env_name'])
env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')
obs_shape = env.observation_space.shape
act_dim = env.action_space.n
self.config['obs_shape'] = obs_shape
self.config['act_dim'] = act_dim
model = AtariModel(act_dim)
algorithm = parl.algorithms.A3C(model,
vf_loss_coeff=config['vf_loss_coeff'])
self.agent = AtariAgent(
algorithm,
obs_shape=self.config['obs_shape'],
predict_thread_num=self.config['predict_thread_num'],
learn_data_provider=self.learn_data_provider)
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_YOU_WANT_TO_USE]` .'
else:
cpu_num = os.environ.get('CPU_NUM')
assert cpu_num is not None and cpu_num == '1', 'Only support training in single CPU,\
Please set environment variable: `export CPU_NUM=1`.'
#========== Learner ==========
self.lr, self.entropy_coeff = None, None
self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.learn_time_stat = TimeStat(100)
self.start_time = None
# learn thread
self.learn_thread = threading.Thread(target=self.run_learn)
self.learn_thread.setDaemon(True)
self.learn_thread.start()
self.predict_input_queue = queue.Queue()
# predict thread
self.predict_threads = []
for i in six.moves.range(self.config['predict_thread_num']):
predict_thread = threading.Thread(target=self.run_predict,
args=(i, ))
predict_thread.setDaemon(True)
predict_thread.start()
self.predict_threads.append(predict_thread)
#========== Remote Simulator ===========
self.remote_count = 0
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.create_actors()
def learn_data_provider(self):
""" Data generator for fluid.layers.py_reader
"""
B = self.config['train_batch_size']
while True:
sample_data = self.sample_data_queue.get()
self.sample_total_steps += len(sample_data['obs'])
for key in sample_data:
self.batch_buffer[key].extend(sample_data[key])
if len(self.batch_buffer['obs']) >= B:
batch = {}
for key in self.batch_buffer:
batch[key] = np.array(self.batch_buffer[key][:B])
obs_np = batch['obs'].astype('float32')
actions_np = batch['actions'].astype('int64')
advantages_np = batch['advantages'].astype('float32')
target_values_np = batch['target_values'].astype('float32')
self.lr = self.lr_scheduler.step()
self.lr = np.array(self.lr, dtype='float32')
self.entropy_coeff = self.entropy_coeff_scheduler.step()
self.entropy_coeff = np.array(self.entropy_coeff,
dtype='float32')
yield [
obs_np, actions_np, advantages_np, target_values_np,
self.lr, self.entropy_coeff
]
for key in self.batch_buffer:
self.batch_buffer[key] = self.batch_buffer[key][B:]
def run_predict(self, thread_id):
""" predict thread
"""
batch_ident = []
batch_obs = []
while True:
ident, obs = self.predict_input_queue.get()
batch_ident.append(ident)
batch_obs.append(obs)
while len(batch_obs) < self.config['max_predict_batch_size']:
try:
ident, obs = self.predict_input_queue.get_nowait()
batch_ident.append(ident)
batch_obs.append(obs)
except queue.Empty:
break
if batch_obs:
batch_obs = np.array(batch_obs)
actions, values = self.agent.sample(batch_obs, thread_id)
for i, ident in enumerate(batch_ident):
self.predict_output_queues[ident].put(
(actions[i], values[i]))
batch_ident = []
batch_obs = []
def run_learn(self):
""" Learn loop
"""
while True:
with self.learn_time_stat:
total_loss, pi_loss, vf_loss, entropy = self.agent.learn()
self.total_loss_stat.add(total_loss)
self.pi_loss_stat.add(pi_loss)
self.vf_loss_stat.add(vf_loss)
self.entropy_stat.add(entropy)
def create_actors(self):
""" Connect to the cluster and start sampling of the remote actor.
"""
parl.connect(self.config['master_address'])
logger.info('Waiting for {} remote actors to connect.'.format(
self.config['actor_num']))
ident = 0
self.predict_output_queues = []
for i in six.moves.range(self.config['actor_num']):
self.remote_count += 1
logger.info('Remote simulator count: {}'.format(self.remote_count))
if self.start_time is None:
self.start_time = time.time()
q = queue.Queue()
self.predict_output_queues.append(q)
remote_thread = threading.Thread(target=self.run_remote_sample,
args=(ident, ))
remote_thread.setDaemon(True)
remote_thread.start()
ident += 1
def run_remote_sample(self, ident):
""" Interacts with remote simulator.
"""
remote_actor = Actor(self.config)
mem = defaultdict(list)
obs = remote_actor.reset()
while True:
self.predict_input_queue.put((ident, obs))
action, value = self.predict_output_queues[ident].get()
next_obs, reward, done = remote_actor.step(action)
mem['obs'].append(obs)
mem['actions'].append(action)
mem['rewards'].append(reward)
mem['values'].append(value)
if done:
next_value = 0
advantages = calc_gae(mem['rewards'], mem['values'],
next_value, self.config['gamma'],
self.config['lambda'])
target_values = advantages + mem['values']
self.sample_data_queue.put({
'obs': mem['obs'],
'actions': mem['actions'],
'advantages': advantages,
'target_values': target_values
})
mem = defaultdict(list)
next_obs = remote_actor.reset()
elif len(mem['obs']) == self.config['t_max'] + 1:
next_value = mem['values'][-1]
advantages = calc_gae(mem['rewards'][:-1], mem['values'][:-1],
next_value, self.config['gamma'],
self.config['lambda'])
target_values = advantages + mem['values'][:-1]
self.sample_data_queue.put({
'obs': mem['obs'][:-1],
'actions': mem['actions'][:-1],
'advantages': advantages,
'target_values': target_values
})
for key in mem:
mem[key] = [mem[key][-1]]
obs = next_obs
if done:
metrics = remote_actor.get_metrics()
if metrics:
self.remote_metrics_queue.put(metrics)
def log_metrics(self):
""" Log metrics of learner and simulators
"""
if self.start_time is None:
return
metrics = []
while True:
try:
metric = self.remote_metrics_queue.get_nowait()
metrics.append(metric)
except queue.Empty:
break
episode_rewards, episode_steps = [], []
for x in metrics:
episode_rewards.extend(x['episode_rewards'])
episode_steps.extend(x['episode_steps'])
max_episode_rewards, mean_episode_rewards, min_episode_rewards, \
max_episode_steps, mean_episode_steps, min_episode_steps =\
None, None, None, None, None, None
if episode_rewards:
mean_episode_rewards = np.mean(np.array(episode_rewards).flatten())
max_episode_rewards = np.max(np.array(episode_rewards).flatten())
min_episode_rewards = np.min(np.array(episode_rewards).flatten())
mean_episode_steps = np.mean(np.array(episode_steps).flatten())
max_episode_steps = np.max(np.array(episode_steps).flatten())
min_episode_steps = np.min(np.array(episode_steps).flatten())
metric = {
'Sample steps': self.sample_total_steps,
'max_episode_rewards': max_episode_rewards,
'mean_episode_rewards': mean_episode_rewards,
'min_episode_rewards': min_episode_rewards,
'max_episode_steps': max_episode_steps,
'mean_episode_steps': mean_episode_steps,
'min_episode_steps': min_episode_steps,
'total_loss': self.total_loss_stat.mean,
'pi_loss': self.pi_loss_stat.mean,
'vf_loss': self.vf_loss_stat.mean,
'entropy': self.entropy_stat.mean,
'learn_time_s': self.learn_time_stat.mean,
'elapsed_time_s': int(time.time() - self.start_time),
'lr': self.lr,
'entropy_coeff': self.entropy_coeff,
}
for key, value in metric.items():
if value is not None:
summary.add_scalar(key, value, self.sample_total_steps)
logger.info(metric)
class Learner(object):
def __init__(self, config):
self.config = config
self.sample_data_queue = queue.Queue(
maxsize=config['sample_queue_max_size'])
#=========== Create Agent ==========
env = gym.make(config['env_name'])
env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')
obs_shape = env.observation_space.shape
act_dim = env.action_space.n
model = AtariModel(act_dim)
algorithm = parl.algorithms.IMPALA(
model,
sample_batch_steps=self.config['sample_batch_steps'],
gamma=self.config['gamma'],
vf_loss_coeff=self.config['vf_loss_coeff'],
clip_rho_threshold=self.config['clip_rho_threshold'],
clip_pg_rho_threshold=self.config['clip_pg_rho_threshold'])
self.agent = AtariAgent(algorithm, obs_shape, act_dim,
self.learn_data_provider)
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_TO_USE]` .'
self.cache_params = self.agent.get_weights()
self.params_lock = threading.Lock()
self.params_updated = False
self.cache_params_sent_cnt = 0
self.total_params_sync = 0
#========== Learner ==========
self.lr, self.entropy_coeff = None, None
self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.kl_stat = WindowStat(100)
self.learn_time_stat = TimeStat(100)
self.start_time = None
self.learn_thread = threading.Thread(target=self.run_learn)
self.learn_thread.setDaemon(True)
self.learn_thread.start()
#========== Remote Actor ===========
self.remote_count = 0
self.batch_buffer = []
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.create_actors()
def learn_data_provider(self):
""" Data generator for fluid.layers.py_reader
"""
while True:
sample_data = self.sample_data_queue.get()
self.sample_total_steps += sample_data['obs'].shape[0]
self.batch_buffer.append(sample_data)
buffer_size = sum(
[data['obs'].shape[0] for data in self.batch_buffer])
if buffer_size >= self.config['train_batch_size']:
batch = {}
for key in self.batch_buffer[0].keys():
batch[key] = np.concatenate(
[data[key] for data in self.batch_buffer])
self.batch_buffer = []
obs_np = batch['obs'].astype('float32')
actions_np = batch['actions'].astype('int64')
behaviour_logits_np = batch['behaviour_logits'].astype(
'float32')
rewards_np = batch['rewards'].astype('float32')
dones_np = batch['dones'].astype('float32')
self.lr = self.lr_scheduler.step()
self.entropy_coeff = self.entropy_coeff_scheduler.step()
yield [
obs_np, actions_np, behaviour_logits_np, rewards_np,
dones_np,
np.float32(self.lr),
np.array([self.entropy_coeff], dtype='float32')
]
def run_learn(self):
""" Learn loop
"""
while True:
with self.learn_time_stat:
total_loss, pi_loss, vf_loss, entropy, kl = self.agent.learn()
self.params_updated = True
self.total_loss_stat.add(total_loss)
self.pi_loss_stat.add(pi_loss)
self.vf_loss_stat.add(vf_loss)
self.entropy_stat.add(entropy)
self.kl_stat.add(kl)
def create_actors(self):
""" Connect to the cluster and start sampling of the remote actor.
"""
parl.connect(self.config['master_address'])
logger.info('Waiting for {} remote actors to connect.'.format(
self.config['actor_num']))
for i in range(self.config['actor_num']):
self.remote_count += 1
logger.info('Remote actor count: {}'.format(self.remote_count))
if self.start_time is None:
self.start_time = time.time()
remote_thread = threading.Thread(target=self.run_remote_sample)
remote_thread.setDaemon(True)
remote_thread.start()
def run_remote_sample(self):
""" Sample data from remote actor and update parameters of remote actor.
"""
remote_actor = Actor(self.config)
cnt = 0
remote_actor.set_weights(self.cache_params)
while True:
batch = remote_actor.sample()
self.sample_data_queue.put(batch)
cnt += 1
if cnt % self.config['get_remote_metrics_interval'] == 0:
metrics = remote_actor.get_metrics()
if metrics:
self.remote_metrics_queue.put(metrics)
self.params_lock.acquire()
if self.params_updated and self.cache_params_sent_cnt >= self.config[
'params_broadcast_interval']:
self.params_updated = False
self.cache_params = self.agent.get_weights()
self.cache_params_sent_cnt = 0
self.cache_params_sent_cnt += 1
self.total_params_sync += 1
self.params_lock.release()
remote_actor.set_weights(self.cache_params)
def log_metrics(self):
""" Log metrics of learner and actors
"""
if self.start_time is None:
return
metrics = []
while True:
try:
metric = self.remote_metrics_queue.get_nowait()
metrics.append(metric)
except queue.Empty:
break
episode_rewards, episode_steps = [], []
for x in metrics:
episode_rewards.extend(x['episode_rewards'])
episode_steps.extend(x['episode_steps'])
max_episode_rewards, mean_episode_rewards, min_episode_rewards, \
max_episode_steps, mean_episode_steps, min_episode_steps =\
None, None, None, None, None, None
if episode_rewards:
mean_episode_rewards = np.mean(np.array(episode_rewards).flatten())
max_episode_rewards = np.max(np.array(episode_rewards).flatten())
min_episode_rewards = np.min(np.array(episode_rewards).flatten())
mean_episode_steps = np.mean(np.array(episode_steps).flatten())
max_episode_steps = np.max(np.array(episode_steps).flatten())
min_episode_steps = np.min(np.array(episode_steps).flatten())
metric = {
'sample_steps': self.sample_total_steps,
'max_episode_rewards': max_episode_rewards,
'mean_episode_rewards': mean_episode_rewards,
'min_episode_rewards': min_episode_rewards,
'max_episode_steps': max_episode_steps,
'mean_episode_steps': mean_episode_steps,
'min_episode_steps': min_episode_steps,
'sample_queue_size': self.sample_data_queue.qsize(),
'total_params_sync': self.total_params_sync,
'cache_params_sent_cnt': self.cache_params_sent_cnt,
'total_loss': self.total_loss_stat.mean,
'pi_loss': self.pi_loss_stat.mean,
'vf_loss': self.vf_loss_stat.mean,
'entropy': self.entropy_stat.mean,
'kl': self.kl_stat.mean,
'learn_time_s': self.learn_time_stat.mean,
'elapsed_time_s': int(time.time() - self.start_time),
'lr': self.lr,
'entropy_coeff': self.entropy_coeff,
}
for key, value in metric.items():
if value is not None:
summary.add_scalar(key, value, self.sample_total_steps)
logger.info(metric)
class Agent(parl.Agent):
def __init__(self, algorithm, config, obs_dim):
"""
Args:
algorithm (`parl.Algorithm`): 强学习算法
config (dict): 配置文件参数
"""
self.obs_dim = obs_dim
super(Agent, self).__init__(algorithm)
# 学习率衰减
self.lr_scheduler = LinearDecayScheduler(config['start_lr'], config['max_sample_steps'])
self.entropy_coeff_scheduler = PiecewiseScheduler(config['entropy_coeff_scheduler'])
# 创建PaddlePaddle程序
def build_program(self):
self.sample_program = fluid.Program()
self.predict_program = fluid.Program()
self.value_program = fluid.Program()
self.learn_program = fluid.Program()
# 给Actor生成数据的程序
with fluid.program_guard(self.sample_program):
obs = layers.data(name='obs', shape=self.obs_dim, dtype='float32')
sample_actions, values = self.alg.sample(obs)
self.sample_outputs = [sample_actions, values]
# 用于预测的程序
with fluid.program_guard(self.predict_program):
obs = layers.data(name='obs', shape=self.obs_dim, dtype='float32')
self.predict_actions = self.alg.predict(obs)
with fluid.program_guard(self.value_program):
obs = layers.data(name='obs', shape=self.obs_dim, dtype='float32')
self.values = self.alg.value(obs)
# 用于训练的程序
with fluid.program_guard(self.learn_program):
obs = layers.data(name='obs', shape=self.obs_dim, dtype='float32')
actions = layers.data(name='actions', shape=[], dtype='int64')
advantages = layers.data(name='advantages', shape=[], dtype='float32')
target_values = layers.data(name='target_values', shape=[], dtype='float32')
lr = layers.data(name='lr', shape=[1], dtype='float32', append_batch_size=False)
entropy_coeff = layers.data(name='entropy_coeff',
shape=[1],
dtype='float32',
append_batch_size=False)
total_loss, pi_loss, vf_loss, entropy = self.alg.learn(
obs, actions, advantages, target_values, lr, entropy_coeff)
self.learn_outputs = [total_loss, pi_loss, vf_loss, entropy]
# 获取并行计算程序
self.learn_program = parl.compile(self.learn_program, total_loss)
def sample(self, obs_np):
"""
Args:
obs_np: 游戏的图像，shape为([N] + observation_space).
游戏图像通道顺序为NCHW.
Returns:
sample_ids: 游戏动作，类型为int64，shape为[N]
values: 模型输出的值，类型为float32，shape为[N]
"""
obs_np = obs_np.astype('float32')
sample_actions, values = self.fluid_executor.run(program=self.sample_program,
feed={'obs': obs_np},
fetch_list=self.sample_outputs)
return sample_actions, values
def predict(self, obs_np):
"""
Args:
obs_np: 游戏的图像，shape为([N] + observation_space).
游戏图像通道顺序为NCHW.
Returns:
sample_ids: 游戏动作，类型为int64，shape为[N]
"""
obs_np = obs_np.astype('float32')
predict_actions = self.fluid_executor.run(program=self.predict_program,
feed={'obs': obs_np},
fetch_list=[self.predict_actions])[0]
return predict_actions
def value(self, obs_np):
"""
Args:
obs_np: 游戏的图像，shape为([N] + observation_space).
游戏图像通道顺序为NCHW.
Returns:
values: 模型输出的值，类型为float32，shape为[N]
"""
obs_np = obs_np.astype('float32')
values = self.fluid_executor.run(program=self.value_program,
feed={'obs': obs_np},
fetch_list=[self.values])[0]
return values
# 执行模型学习
def learn(self, obs_np, actions_np, advantages_np, target_values_np):
"""
Args:
obs_np: 游戏的图像，shape为([N] + observation_space).
游戏图像通道顺序为NCHW.
actions_np: 游戏动作，类型为int64，shape为[N]
advantages_np: 奖励值，类型为float32，shape为[N]
target_values_np: 目标模型值，类型为float32，shape为[N]
"""
obs_np = obs_np.astype('float32')
actions_np = actions_np.astype('int64')
advantages_np = advantages_np.astype('float32')
target_values_np = target_values_np.astype('float32')
lr = self.lr_scheduler.step(step_num=obs_np.shape[0])
entropy_coeff = self.entropy_coeff_scheduler.step()
total_loss, pi_loss, vf_loss, entropy = self.fluid_executor.run(
self.learn_program,
feed={
'obs': obs_np,
'actions': actions_np,
'advantages': advantages_np,
'target_values': target_values_np,
'lr': np.array([lr], dtype='float32'),
'entropy_coeff': np.array([entropy_coeff], dtype='float32')
},
fetch_list=self.learn_outputs)
return total_loss, pi_loss, vf_loss, entropy, lr, entropy_coeff
class AtariAgent(parl.Agent):
def __init__(self, algorithm, config):
"""
Args:
algorithm (`parl.Algorithm`): algorithm to be used in this agent.
config (dict): config file describing the training hyper-parameters(see a2c_config.py)
"""
self.obs_shape = config['obs_shape']
super(AtariAgent, self).__init__(algorithm)
self.lr_scheduler = LinearDecayScheduler(config['start_lr'],
config['max_sample_steps'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
def build_program(self):
self.sample_program = fluid.Program()
self.predict_program = fluid.Program()
self.value_program = fluid.Program()
self.learn_program = fluid.Program()
with fluid.program_guard(self.sample_program):
obs = layers.data(name='obs',
shape=self.obs_shape,
dtype='float32')
sample_actions, values = self.alg.sample(obs)
self.sample_outputs = [sample_actions, values]
with fluid.program_guard(self.predict_program):
obs = layers.data(name='obs',
shape=self.obs_shape,
dtype='float32')
self.predict_actions = self.alg.predict(obs)
with fluid.program_guard(self.value_program):
obs = layers.data(name='obs',
shape=self.obs_shape,
dtype='float32')
self.values = self.alg.value(obs)
with fluid.program_guard(self.learn_program):
obs = layers.data(name='obs',
shape=self.obs_shape,
dtype='float32')
actions = layers.data(name='actions', shape=[], dtype='int64')
advantages = layers.data(name='advantages',
shape=[],
dtype='float32')
target_values = layers.data(name='target_values',
shape=[],
dtype='float32')
lr = layers.data(name='lr',
shape=[1],
dtype='float32',
append_batch_size=False)
entropy_coeff = layers.data(name='entropy_coeff',
shape=[1],
dtype='float32',
append_batch_size=False)
total_loss, pi_loss, vf_loss, entropy = self.alg.learn(
obs, actions, advantages, target_values, lr, entropy_coeff)
self.learn_outputs = [total_loss, pi_loss, vf_loss, entropy]
self.learn_program = parl.compile(self.learn_program, total_loss)
def sample(self, obs_np):
"""
Args:
obs_np: a numpy float32 array of shape ([B] + observation_space).
Format of image input should be NCHW format.
Returns:
sample_ids: a numpy int64 array of shape [B]
values: a numpy float32 array of shape [B]
"""
obs_np = obs_np.astype('float32')
sample_actions, values = self.fluid_executor.run(
self.sample_program,
feed={'obs': obs_np},
fetch_list=self.sample_outputs)
return sample_actions, values
def predict(self, obs_np):
"""
Args:
obs_np: a numpy float32 array of shape ([B] + observation_space).
Format of image input should be NCHW format.
Returns:
sample_ids: a numpy int64 array of shape [B]
"""
obs_np = obs_np.astype('float32')
predict_actions = self.fluid_executor.run(
self.predict_program,
feed={'obs': obs_np},
fetch_list=[self.predict_actions])[0]
return predict_actions
def value(self, obs_np):
"""
Args:
obs_np: a numpy float32 array of shape ([B] + observation_space).
Format of image input should be NCHW format.
Returns:
values: a numpy float32 array of shape [B]
"""
obs_np = obs_np.astype('float32')
values = self.fluid_executor.run(self.value_program,
feed={'obs': obs_np},
fetch_list=[self.values])[0]
return values
def learn(self, obs_np, actions_np, advantages_np, target_values_np):
"""
Args:
obs_np: a numpy float32 array of shape ([B] + observation_space).
Format of image input should be NCHW format.
actions_np: a numpy int64 array of shape [B]
advantages_np: a numpy float32 array of shape [B]
target_values_np: a numpy float32 array of shape [B]
"""
obs_np = obs_np.astype('float32')
actions_np = actions_np.astype('int64')
advantages_np = advantages_np.astype('float32')
target_values_np = target_values_np.astype('float32')
lr = self.lr_scheduler.step(step_num=obs_np.shape[0])
entropy_coeff = self.entropy_coeff_scheduler.step()
total_loss, pi_loss, vf_loss, entropy = self.fluid_executor.run(
self.learn_program,
feed={
'obs': obs_np,
'actions': actions_np,
'advantages': advantages_np,
'target_values': target_values_np,
'lr': np.array([lr], dtype='float32'),
'entropy_coeff': np.array([entropy_coeff], dtype='float32')
},
fetch_list=self.learn_outputs)
return total_loss, pi_loss, vf_loss, entropy, lr, entropy_coeff
class A2C(parl.Algorithm):
def __init__(self, model, config):
assert isinstance(config['vf_loss_coeff'], (int, float))
self.model = model
self.vf_loss_coeff = config['vf_loss_coeff']
self.optimizer = optim.Adam(self.model.parameters(),
lr=config['learning_rate'])
self.config = config
self.lr_scheduler = LinearDecayScheduler(config['start_lr'],
config['max_sample_steps'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
def learn(self, obs, actions, advantages, target_values):
prob = self.model.policy(obs, softmax_dim=1)
policy_distri = Categorical(prob)
actions_log_probs = policy_distri.log_prob(actions)
# The policy gradient loss
pi_loss = -((actions_log_probs * advantages).sum())
# The value function loss
values = self.model.value(obs).reshape(-1)
delta = values - target_values
vf_loss = 0.5 * torch.mul(delta, delta).sum()
# The entropy loss (We want to maximize entropy, so entropy_ceoff < 0)
policy_entropy = policy_distri.entropy()
entropy = policy_entropy.sum()
lr = self.lr_scheduler.step(step_num=obs.shape[0])
entropy_coeff = self.entropy_coeff_scheduler.step()
total_loss = pi_loss + vf_loss * self.vf_loss_coeff + entropy * entropy_coeff
for param_group in self.optimizer.param_groups:
param_group['lr'] = lr
total_loss.backward()
self.optimizer.step()
self.optimizer.zero_grad()
return total_loss, pi_loss, vf_loss, entropy, lr, entropy_coeff
def sample(self, obs):
prob, values = self.model.policy_and_value(obs)
sample_actions = Categorical(prob).sample()
return sample_actions, values
def predict(self, obs):
prob = self.model.policy(obs)
_, predict_actions = prob.max(-1)
return predict_actions
def value(self, obs):
values = self.model.value(obs)
return values
class Learner(object):
def __init__(self, config):
self.config = config
self.sample_data_queue = queue.Queue(
maxsize=config['sample_queue_max_size'])
#=========== Create Agent ==========
env = IntraBuildingEnv("config.ini")
self._mansion_attr = env._mansion.attribute
self._obs_dim = obs_dim(self._mansion_attr)
self._act_dim = act_dim(self._mansion_attr)
self.config['obs_shape'] = self._obs_dim
self.config['act_dim'] = self._act_dim
model = RLDispatcherModel(self._act_dim)
algorithm = IMPALA(model, hyperparas=config)
self.agent = ElevatorAgent(algorithm, config, self.learn_data_provider)
self.cache_params = self.agent.get_params()
self.params_lock = threading.Lock()
self.params_updated = False
self.cache_params_sent_cnt = 0
self.total_params_sync = 0
#========== Learner ==========
self.lr, self.entropy_coeff = None, None
self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.kl_stat = WindowStat(100)
self.learn_time_stat = TimeStat(100)
self.start_time = None
self.learn_thread = threading.Thread(target=self.run_learn)
self.learn_thread.setDaemon(True)
self.learn_thread.start()
#========== Remote Actor ===========
self.remote_count = 0
self.batch_buffer = []
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.remote_manager_thread = threading.Thread(
target=self.run_remote_manager)
self.remote_manager_thread.setDaemon(True)
self.remote_manager_thread.start()
self.csv_logger = CSVLogger(
os.path.join(logger.get_dir(), 'result.csv'))
from utils import Summary
self.summary = Summary('./output')
def learn_data_provider(self):
""" Data generator for fluid.layers.py_reader
"""
while True:
sample_data = self.sample_data_queue.get()
self.sample_total_steps += sample_data['obs'].shape[0]
self.batch_buffer.append(sample_data)
buffer_size = sum(
[data['obs'].shape[0] for data in self.batch_buffer])
if buffer_size >= self.config['train_batch_size']:
batch = {}
for key in self.batch_buffer[0].keys():
batch[key] = np.concatenate(
[data[key] for data in self.batch_buffer])
self.batch_buffer = []
obs_np = batch['obs'].astype('float32')
actions_np = batch['actions'].astype('int64')
behaviour_logits_np = batch['behaviour_logits'].astype(
'float32')
rewards_np = batch['rewards'].astype('float32')
dones_np = batch['dones'].astype('float32')
self.lr = self.lr_scheduler.step()
self.entropy_coeff = self.entropy_coeff_scheduler.step()
yield [
obs_np, actions_np, behaviour_logits_np, rewards_np,
dones_np, self.lr, self.entropy_coeff
]
def run_learn(self):
""" Learn loop
"""
while True:
with self.learn_time_stat:
total_loss, pi_loss, vf_loss, entropy, kl = self.agent.learn()
self.params_updated = True
self.total_loss_stat.add(total_loss)
self.pi_loss_stat.add(pi_loss)
self.vf_loss_stat.add(vf_loss)
self.entropy_stat.add(entropy)
self.kl_stat.add(kl)
def run_remote_manager(self):
""" Accept connection of new remote actor and start sampling of the remote actor.
"""
remote_manager = RemoteManager(port=self.config['server_port'])
logger.info('Waiting for remote actors connecting.')
while True:
remote_actor = remote_manager.get_remote()
self.remote_count += 1
logger.info('Remote actor count: {}'.format(self.remote_count))
if self.start_time is None:
self.start_time = time.time()
remote_thread = threading.Thread(
target=self.run_remote_sample, args=(remote_actor, ))
remote_thread.setDaemon(True)
remote_thread.start()
def run_remote_sample(self, remote_actor):
""" Sample data from remote actor and update parameters of remote actor.
"""
cnt = 0
remote_actor.set_params(self.cache_params)
while True:
batch = remote_actor.sample()
if batch:
self.sample_data_queue.put(batch)
cnt += 1
if cnt % self.config['get_remote_metrics_interval'] == 0:
metrics = remote_actor.get_metrics()
if metrics:
self.remote_metrics_queue.put(metrics)
self.params_lock.acquire()
if self.params_updated and self.cache_params_sent_cnt >= self.config[
'params_broadcast_interval']:
self.params_updated = False
self.cache_params = self.agent.get_params()
self.cache_params_sent_cnt = 0
self.cache_params_sent_cnt += 1
self.total_params_sync += 1
self.params_lock.release()
remote_actor.set_params(self.cache_params)
def log_metrics(self):
""" Log metrics of learner and actors
"""
if self.start_time is None:
return
metrics = []
while True:
try:
metric = self.remote_metrics_queue.get_nowait()
metrics.append(metric)
except queue.Empty:
break
episode_rewards, episode_steps, episode_shaping_rewards, episode_deliver_rewards, episode_wrong_deliver_rewards = [], [], [], [], []
for x in metrics:
episode_rewards.extend(x['episode_rewards'])
episode_steps.extend(x['episode_steps'])
episode_shaping_rewards.extend(x['episode_shaping_rewards'])
episode_deliver_rewards.extend(x['episode_deliver_rewards'])
episode_wrong_deliver_rewards.extend(x['episode_wrong_deliver_rewards'])
max_episode_rewards, mean_episode_rewards, min_episode_rewards, \
max_episode_steps, mean_episode_steps, min_episode_steps =\
None, None, None, None, None, None
mean_episode_shaping_rewards, mean_episode_deliver_rewards, mean_episode_wrong_deliver_rewards = \
None, None, None
if episode_rewards:
mean_episode_rewards = np.mean(np.array(episode_rewards).flatten())
max_episode_rewards = np.max(np.array(episode_rewards).flatten())
min_episode_rewards = np.min(np.array(episode_rewards).flatten())
mean_episode_steps = np.mean(np.array(episode_steps).flatten())
max_episode_steps = np.max(np.array(episode_steps).flatten())
min_episode_steps = np.min(np.array(episode_steps).flatten())
mean_episode_shaping_rewards = np.mean(np.array(episode_shaping_rewards).flatten())
mean_episode_deliver_rewards = np.mean(np.array(episode_deliver_rewards).flatten())
mean_episode_wrong_deliver_rewards = np.mean(np.array(episode_wrong_deliver_rewards).flatten())
#print(self.learn_time_stat.time_samples.items)
metric = {
'Sample steps': self.sample_total_steps,
'max_episode_rewards': max_episode_rewards,
'mean_episode_rewards': mean_episode_rewards,
'min_episode_rewards': min_episode_rewards,
'mean_episode_shaping_rewards': mean_episode_shaping_rewards,
'mean_episode_deliver_rewards': mean_episode_deliver_rewards,
'mean_episode_wrong_deliver_rewards': mean_episode_wrong_deliver_rewards,
#'max_episode_steps': max_episode_steps,
#'mean_episode_steps': mean_episode_steps,
#'min_episode_steps': min_episode_steps,
'sample_queue_size': self.sample_data_queue.qsize(),
'total_params_sync': self.total_params_sync,
'cache_params_sent_cnt': self.cache_params_sent_cnt,
'total_loss': self.total_loss_stat.mean,
'pi_loss': self.pi_loss_stat.mean,
'vf_loss': self.vf_loss_stat.mean,
'entropy': self.entropy_stat.mean,
'kl': self.kl_stat.mean,
'learn_time_s': self.learn_time_stat.mean,
'elapsed_time_s': int(time.time() - self.start_time),
'lr': self.lr,
'entropy_coeff': self.entropy_coeff,
}
logger.info(metric)
self.csv_logger.log_dict(metric)
self.summary.log_dict(metric, self.sample_total_steps)
def close(self):
self.csv_logger.close()