class TimeStat(object):
"""A time stat for logging the elapsed time of code running
Example:
time_stat = TimeStat()
with time_stat:
// some code
print(time_stat.mean)
"""
def __init__(self, window_size=1):
self.time_samples = WindowStat(window_size)
self._start_time = None
def __enter__(self):
self._start_time = time.time()
def __exit__(self, type, value, tb):
time_delta = time.time() - self._start_time
self.time_samples.add(time_delta)
@property
def mean(self):
return self.time_samples.mean
@property
def min(self):
return self.time_samples.min
@property
def max(self):
return self.time_samples.max
def __init__(self, config):
self.config = config
env = gym.make(self.config['env_name'])
self.config['obs_dim'] = env.observation_space.shape[0]
self.config['act_dim'] = env.action_space.shape[0]
self.obs_filter = MeanStdFilter(self.config['obs_dim'])
self.noise = SharedNoiseTable(self.config['noise_size'])
model = MujocoModel(self.config['act_dim'])
algorithm = ES(model)
self.agent = MujocoAgent(algorithm, self.config)
self.latest_flat_weights = self.agent.get_flat_weights()
self.latest_obs_filter = self.obs_filter.as_serializable()
self.sample_total_episodes = 0
self.sample_total_steps = 0
self.actors_signal_input_queues = []
self.actors_output_queues = []
self.create_actors()
self.eval_rewards_stat = WindowStat(self.config['report_window_size'])
self.eval_lengths_stat = WindowStat(self.config['report_window_size'])
def __init__(self, config):
self.config = config
self.sample_data_queue = queue.Queue(
maxsize=config['sample_queue_max_size'])
#=========== Create Agent ==========
env = gym.make(config['env_name'])
env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')
obs_shape = env.observation_space.shape
act_dim = env.action_space.n
model = AtariModel(act_dim)
algorithm = parl.algorithms.IMPALA(
model,
sample_batch_steps=self.config['sample_batch_steps'],
gamma=self.config['gamma'],
vf_loss_coeff=self.config['vf_loss_coeff'],
clip_rho_threshold=self.config['clip_rho_threshold'],
clip_pg_rho_threshold=self.config['clip_pg_rho_threshold'])
self.agent = AtariAgent(algorithm, obs_shape, act_dim,
self.learn_data_provider)
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_TO_USE]` .'
self.cache_params = self.agent.get_weights()
self.params_lock = threading.Lock()
self.params_updated = False
self.cache_params_sent_cnt = 0
self.total_params_sync = 0
#========== Learner ==========
self.lr, self.entropy_coeff = None, None
self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.kl_stat = WindowStat(100)
self.learn_time_stat = TimeStat(100)
self.start_time = None
self.learn_thread = threading.Thread(target=self.run_learn)
self.learn_thread.setDaemon(True)
self.learn_thread.start()
#========== Remote Actor ===========
self.remote_count = 0
self.batch_buffer = []
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.create_actors()
def __init__(self, config):
self.config = config
self.sample_data_queue = queue.Queue(
maxsize=config['sample_queue_max_size'])
#=========== Create Agent ==========
env = IntraBuildingEnv("config.ini")
self._mansion_attr = env._mansion.attribute
self._obs_dim = obs_dim(self._mansion_attr)
self._act_dim = act_dim(self._mansion_attr)
self.config['obs_shape'] = self._obs_dim
self.config['act_dim'] = self._act_dim
model = RLDispatcherModel(self._act_dim)
algorithm = IMPALA(model, hyperparas=config)
self.agent = ElevatorAgent(algorithm, config, self.learn_data_provider)
self.cache_params = self.agent.get_params()
self.params_lock = threading.Lock()
self.params_updated = False
self.cache_params_sent_cnt = 0
self.total_params_sync = 0
#========== Learner ==========
self.lr, self.entropy_coeff = None, None
self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.kl_stat = WindowStat(100)
self.learn_time_stat = TimeStat(100)
self.start_time = None
self.learn_thread = threading.Thread(target=self.run_learn)
self.learn_thread.setDaemon(True)
self.learn_thread.start()
#========== Remote Actor ===========
self.remote_count = 0
self.batch_buffer = []
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.remote_manager_thread = threading.Thread(
target=self.run_remote_manager)
self.remote_manager_thread.setDaemon(True)
self.remote_manager_thread.start()
self.csv_logger = CSVLogger(
os.path.join(logger.get_dir(), 'result.csv'))
from utils import Summary
self.summary = Summary('./output')
def __init__(self, config):
self.config = config
# 这里创建游戏单纯是为了获取游戏动作的维度
env = retro_util.RetroEnv(game=config['env_name'],
use_restricted_actions=retro.Actions.DISCRETE,
resize_shape=config['obs_shape'],
render_preprocess=False)
obs_dim = env.observation_space.shape
action_dim = env.action_space.n
self.config['action_dim'] = action_dim
# 这里创建的模型是真正学习使用的
model = Model(action_dim)
algorithm = parl.algorithms.A3C(model, vf_loss_coeff=config['vf_loss_coeff'])
self.agent = Agent(algorithm, config, obs_dim)
# 只支持单个GPU
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_TO_USE]` .'
# 加载预训练模型
if self.config['restore_model']:
logger.info("加载预训练模型...")
self.agent.restore(self.config['model_path'])
# 记录训练的日志
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.lr = None
self.entropy_coeff = None
self.best_loss = None
self.learn_time_stat = TimeStat(100)
self.start_time = None
# ========== Remote Actor ===========
self.remote_count = 0
self.sample_data_queue = queue.Queue()
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.params_queues = []
self.create_actors()
def __init__(self, config, cuda):
self.cuda = cuda
self.config = config
env = gym.make(config['env_name'])
env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')
obs_shape = env.observation_space.shape
act_dim = env.action_space.n
self.config['obs_shape'] = obs_shape
self.config['act_dim'] = act_dim
model = ActorCritic(act_dim)
if self.cuda:
model = model.cuda()
algorithm = A2C(model, config)
self.agent = Agent(algorithm, config)
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_YOU_WANT_TO_USE]` .'
else:
os.environ['CPU_NUM'] = str(1)
#========== Learner ==========
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.lr = None
self.entropy_coeff = None
self.learn_time_stat = TimeStat(100)
self.start_time = None
#========== Remote Actor ===========
self.remote_count = 0
self.sample_total_steps = 0
self.sample_data_queue = queue.Queue()
self.remote_metrics_queue = queue.Queue()
self.params_queues = []
self.create_actors()
def __init__(self, args):
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_TO_USE]` .'
else:
cpu_num = os.environ.get('CPU_NUM')
assert cpu_num is not None and cpu_num == '1', 'Only support training in single CPU,\
Please set environment variable: `export CPU_NUM=1`.'
model = OpenSimModel(OBS_DIM, VEL_DIM, ACT_DIM)
algorithm = parl.algorithms.DDPG(
model,
gamma=GAMMA,
tau=TAU,
actor_lr=ACTOR_LR,
critic_lr=CRITIC_LR)
self.agent = OpenSimAgent(algorithm, OBS_DIM, ACT_DIM)
self.rpm = ReplayMemory(args.rpm_size, OBS_DIM, ACT_DIM)
if args.restore_rpm_path is not None:
self.rpm.load(args.restore_rpm_path)
if args.restore_model_path is not None:
self.restore(args.restore_model_path)
# add lock between training and predicting
self.model_lock = threading.Lock()
# add lock when appending data to rpm or writing scalars to summary
self.memory_lock = threading.Lock()
self.ready_actor_queue = queue.Queue()
self.total_steps = 0
self.noiselevel = 0.5
self.critic_loss_stat = WindowStat(500)
self.env_reward_stat = WindowStat(500)
self.shaping_reward_stat = WindowStat(500)
self.max_env_reward = 0
# thread to keep training
learn_thread = threading.Thread(target=self.keep_training)
learn_thread.setDaemon(True)
learn_thread.start()
self.create_actors()
class Learner(object):
def __init__(self, config):
self.config = config
self.sample_data_queue = queue.Queue(
maxsize=config['sample_queue_max_size'])
#=========== Create Agent ==========
env = gym.make(config['env_name'])
env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')
obs_shape = env.observation_space.shape
act_dim = env.action_space.n
model = AtariModel(act_dim)
algorithm = parl.algorithms.IMPALA(
model,
sample_batch_steps=self.config['sample_batch_steps'],
gamma=self.config['gamma'],
vf_loss_coeff=self.config['vf_loss_coeff'],
clip_rho_threshold=self.config['clip_rho_threshold'],
clip_pg_rho_threshold=self.config['clip_pg_rho_threshold'])
self.agent = AtariAgent(algorithm, obs_shape, act_dim,
self.learn_data_provider)
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_TO_USE]` .'
self.cache_params = self.agent.get_weights()
self.params_lock = threading.Lock()
self.params_updated = False
self.cache_params_sent_cnt = 0
self.total_params_sync = 0
#========== Learner ==========
self.lr, self.entropy_coeff = None, None
self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.kl_stat = WindowStat(100)
self.learn_time_stat = TimeStat(100)
self.start_time = None
self.learn_thread = threading.Thread(target=self.run_learn)
self.learn_thread.setDaemon(True)
self.learn_thread.start()
#========== Remote Actor ===========
self.remote_count = 0
self.batch_buffer = []
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.create_actors()
def learn_data_provider(self):
""" Data generator for fluid.layers.py_reader
"""
while True:
sample_data = self.sample_data_queue.get()
self.sample_total_steps += sample_data['obs'].shape[0]
self.batch_buffer.append(sample_data)
buffer_size = sum(
[data['obs'].shape[0] for data in self.batch_buffer])
if buffer_size >= self.config['train_batch_size']:
batch = {}
for key in self.batch_buffer[0].keys():
batch[key] = np.concatenate(
[data[key] for data in self.batch_buffer])
self.batch_buffer = []
obs_np = batch['obs'].astype('float32')
actions_np = batch['actions'].astype('int64')
behaviour_logits_np = batch['behaviour_logits'].astype(
'float32')
rewards_np = batch['rewards'].astype('float32')
dones_np = batch['dones'].astype('float32')
self.lr = self.lr_scheduler.step()
self.entropy_coeff = self.entropy_coeff_scheduler.step()
yield [
obs_np, actions_np, behaviour_logits_np, rewards_np,
dones_np,
np.float32(self.lr),
np.array([self.entropy_coeff], dtype='float32')
]
def run_learn(self):
""" Learn loop
"""
while True:
with self.learn_time_stat:
total_loss, pi_loss, vf_loss, entropy, kl = self.agent.learn()
self.params_updated = True
self.total_loss_stat.add(total_loss)
self.pi_loss_stat.add(pi_loss)
self.vf_loss_stat.add(vf_loss)
self.entropy_stat.add(entropy)
self.kl_stat.add(kl)
def create_actors(self):
""" Connect to the cluster and start sampling of the remote actor.
"""
parl.connect(self.config['master_address'])
logger.info('Waiting for {} remote actors to connect.'.format(
self.config['actor_num']))
for i in range(self.config['actor_num']):
self.remote_count += 1
logger.info('Remote actor count: {}'.format(self.remote_count))
if self.start_time is None:
self.start_time = time.time()
remote_thread = threading.Thread(target=self.run_remote_sample)
remote_thread.setDaemon(True)
remote_thread.start()
def run_remote_sample(self):
""" Sample data from remote actor and update parameters of remote actor.
"""
remote_actor = Actor(self.config)
cnt = 0
remote_actor.set_weights(self.cache_params)
while True:
batch = remote_actor.sample()
self.sample_data_queue.put(batch)
cnt += 1
if cnt % self.config['get_remote_metrics_interval'] == 0:
metrics = remote_actor.get_metrics()
if metrics:
self.remote_metrics_queue.put(metrics)
self.params_lock.acquire()
if self.params_updated and self.cache_params_sent_cnt >= self.config[
'params_broadcast_interval']:
self.params_updated = False
self.cache_params = self.agent.get_weights()
self.cache_params_sent_cnt = 0
self.cache_params_sent_cnt += 1
self.total_params_sync += 1
self.params_lock.release()
remote_actor.set_weights(self.cache_params)
def log_metrics(self):
""" Log metrics of learner and actors
"""
if self.start_time is None:
return
metrics = []
while True:
try:
metric = self.remote_metrics_queue.get_nowait()
metrics.append(metric)
except queue.Empty:
break
episode_rewards, episode_steps = [], []
for x in metrics:
episode_rewards.extend(x['episode_rewards'])
episode_steps.extend(x['episode_steps'])
max_episode_rewards, mean_episode_rewards, min_episode_rewards, \
max_episode_steps, mean_episode_steps, min_episode_steps =\
None, None, None, None, None, None
if episode_rewards:
mean_episode_rewards = np.mean(np.array(episode_rewards).flatten())
max_episode_rewards = np.max(np.array(episode_rewards).flatten())
min_episode_rewards = np.min(np.array(episode_rewards).flatten())
mean_episode_steps = np.mean(np.array(episode_steps).flatten())
max_episode_steps = np.max(np.array(episode_steps).flatten())
min_episode_steps = np.min(np.array(episode_steps).flatten())
metric = {
'sample_steps': self.sample_total_steps,
'max_episode_rewards': max_episode_rewards,
'mean_episode_rewards': mean_episode_rewards,
'min_episode_rewards': min_episode_rewards,
'max_episode_steps': max_episode_steps,
'mean_episode_steps': mean_episode_steps,
'min_episode_steps': min_episode_steps,
'sample_queue_size': self.sample_data_queue.qsize(),
'total_params_sync': self.total_params_sync,
'cache_params_sent_cnt': self.cache_params_sent_cnt,
'total_loss': self.total_loss_stat.mean,
'pi_loss': self.pi_loss_stat.mean,
'vf_loss': self.vf_loss_stat.mean,
'entropy': self.entropy_stat.mean,
'kl': self.kl_stat.mean,
'learn_time_s': self.learn_time_stat.mean,
'elapsed_time_s': int(time.time() - self.start_time),
'lr': self.lr,
'entropy_coeff': self.entropy_coeff,
}
for key, value in metric.items():
if value is not None:
summary.add_scalar(key, value, self.sample_total_steps)
logger.info(metric)
class Learner(object):
def __init__(self, args):
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_TO_USE]` .'
else:
cpu_num = os.environ.get('CPU_NUM')
assert cpu_num is not None and cpu_num == '1', 'Only support training in single CPU,\
Please set environment variable: `export CPU_NUM=1`.'
model = OpenSimModel(OBS_DIM, VEL_DIM, ACT_DIM)
algorithm = parl.algorithms.DDPG(
model,
gamma=GAMMA,
tau=TAU,
actor_lr=ACTOR_LR,
critic_lr=CRITIC_LR)
self.agent = OpenSimAgent(algorithm, OBS_DIM, ACT_DIM)
self.rpm = ReplayMemory(args.rpm_size, OBS_DIM, ACT_DIM)
if args.restore_rpm_path is not None:
self.rpm.load(args.restore_rpm_path)
if args.restore_model_path is not None:
self.restore(args.restore_model_path)
# add lock between training and predicting
self.model_lock = threading.Lock()
# add lock when appending data to rpm or writing scalars to summary
self.memory_lock = threading.Lock()
self.ready_actor_queue = queue.Queue()
self.total_steps = 0
self.noiselevel = 0.5
self.critic_loss_stat = WindowStat(500)
self.env_reward_stat = WindowStat(500)
self.shaping_reward_stat = WindowStat(500)
self.max_env_reward = 0
# thread to keep training
learn_thread = threading.Thread(target=self.keep_training)
learn_thread.setDaemon(True)
learn_thread.start()
self.create_actors()
def create_actors(self):
"""Connect to the cluster and start sampling of the remote actor.
"""
parl.connect(args.cluster_address, ['official_obs_scaler.npz'])
for i in range(args.actor_num):
logger.info('Remote actor count: {}'.format(i + 1))
remote_thread = threading.Thread(target=self.run_remote_sample)
remote_thread.setDaemon(True)
remote_thread.start()
# There is a memory-leak problem in osim-rl package.
# So we will dynamically add actors when remote actors killed due to excessive memory usage.
time.sleep(10 * 60)
parl_client = get_global_client()
while True:
if parl_client.actor_num < args.actor_num:
logger.info(
'Dynamic adding acotr, current actor num:{}'.format(
parl_client.actor_num))
remote_thread = threading.Thread(target=self.run_remote_sample)
remote_thread.setDaemon(True)
remote_thread.start()
time.sleep(5)
def _new_ready_actor(self):
"""
The actor is ready to start new episode,
but blocking until training thread call actor_ready_event.set()
"""
actor_ready_event = threading.Event()
self.ready_actor_queue.put(actor_ready_event)
logger.info(
"[new_avaliabe_actor] approximate size of ready actors:{}".format(
self.ready_actor_queue.qsize()))
actor_ready_event.wait()
def run_remote_sample(self):
remote_actor = Actor(
difficulty=args.difficulty,
vel_penalty_coeff=args.vel_penalty_coeff,
muscle_penalty_coeff=args.muscle_penalty_coeff,
penalty_coeff=args.penalty_coeff,
only_first_target=args.only_first_target)
actor_state = ActorState()
while True:
obs = remote_actor.reset()
actor_state.reset()
while True:
actor_state.memory.append(
TransitionExperience(
obs=obs,
action=None,
reward=None,
info=None,
timestamp=time.time()))
action = self.pred_batch(obs)
# For each target, decay noise as the steps increase.
step = len(
actor_state.memory) - actor_state.last_target_changed_steps
current_noise = self.noiselevel * (0.98**(step - 1))
noise = np.zeros((ACT_DIM, ), dtype=np.float32)
if actor_state.ident % 3 == 0:
if step % 5 == 0:
noise = np.random.randn(ACT_DIM) * current_noise
elif actor_state.ident % 3 == 1:
if step % 5 == 0:
noise = np.random.randn(ACT_DIM) * current_noise * 2
action += noise
action = np.clip(action, -1, 1)
obs, reward, done, info = remote_actor.step(action)
reward_scale = (1 - GAMMA)
info['shaping_reward'] *= reward_scale
actor_state.memory[-1].reward = reward
actor_state.memory[-1].info = info
actor_state.memory[-1].action = action
if 'target_changed' in info and info['target_changed']:
actor_state.update_last_target_changed()
if done:
self._parse_memory(actor_state, last_obs=obs)
break
self._new_ready_actor()
def _parse_memory(self, actor_state, last_obs):
mem = actor_state.memory
n = len(mem)
episode_shaping_reward = np.sum(
[exp.info['shaping_reward'] for exp in mem])
episode_env_reward = np.sum([exp.info['env_reward'] for exp in mem])
episode_time = time.time() - mem[0].timestamp
episode_rpm = []
for i in range(n - 1):
episode_rpm.append([
mem[i].obs, mem[i].action, mem[i].info['shaping_reward'],
mem[i + 1].obs, False
])
episode_rpm.append([
mem[-1].obs, mem[-1].action, mem[-1].info['shaping_reward'],
last_obs, not mem[-1].info['timeout']
])
with self.memory_lock:
self.total_steps += n
self.add_episode_rpm(episode_rpm)
if actor_state.ident % 3 == 2: # trajectory without noise
self.env_reward_stat.add(episode_env_reward)
self.shaping_reward_stat.add(episode_shaping_reward)
self.max_env_reward = max(self.max_env_reward,
episode_env_reward)
if self.env_reward_stat.count > 500:
summary.add_scalar('recent_env_reward',
self.env_reward_stat.mean,
self.total_steps)
summary.add_scalar('recent_shaping_reward',
self.shaping_reward_stat.mean,
self.total_steps)
if self.critic_loss_stat.count > 500:
summary.add_scalar('recent_critic_loss',
self.critic_loss_stat.mean,
self.total_steps)
summary.add_scalar('episode_length', n, self.total_steps)
summary.add_scalar('max_env_reward', self.max_env_reward,
self.total_steps)
summary.add_scalar('ready_actor_num',
self.ready_actor_queue.qsize(),
self.total_steps)
summary.add_scalar('episode_time', episode_time,
self.total_steps)
self.noiselevel = self.noiselevel * NOISE_DECAY
def learn(self):
start_time = time.time()
for T in range(args.train_times):
[states, actions, rewards, new_states,
dones] = self.rpm.sample_batch(BATCH_SIZE)
with self.model_lock:
critic_loss = self.agent.learn(states, actions, rewards,
new_states, dones)
self.critic_loss_stat.add(critic_loss)
logger.info(
"[learn] time consuming:{}".format(time.time() - start_time))
def keep_training(self):
episode_count = 1000000
for T in range(episode_count):
if self.rpm.size() > BATCH_SIZE * args.warm_start_batchs:
self.learn()
logger.info(
"[keep_training/{}] trying to acq a new env".format(T))
# Keep training and predicting balance
# After training, wait for a ready actor, and make the actor start new episode
ready_actor_event = self.ready_actor_queue.get()
ready_actor_event.set()
if np.mod(T, 100) == 0:
logger.info("saving models")
self.save(T)
if np.mod(T, 10000) == 0:
logger.info("saving rpm")
self.save_rpm()
def save_rpm(self):
save_path = os.path.join(logger.get_dir(), "rpm.npz")
self.rpm.save(save_path)
def save(self, T):
save_path = os.path.join(
logger.get_dir(), 'model_every_100_episodes/episodes-{}'.format(T))
self.agent.save(save_path)
def restore(self, model_path):
logger.info('restore model from {}'.format(model_path))
self.agent.restore(model_path)
def add_episode_rpm(self, episode_rpm):
for x in episode_rpm:
self.rpm.append(
obs=x[0], act=x[1], reward=x[2], next_obs=x[3], terminal=x[4])
def pred_batch(self, obs):
batch_obs = np.expand_dims(obs, axis=0)
with self.model_lock:
action = self.agent.predict(batch_obs.astype('float32'))
action = np.squeeze(action, axis=0)
return action
class Learner(object):
def __init__(self, config):
self.config = config
env = gym.make(self.config['env_name'])
self.config['obs_dim'] = env.observation_space.shape[0]
self.config['act_dim'] = env.action_space.shape[0]
self.obs_filter = MeanStdFilter(self.config['obs_dim'])
self.noise = SharedNoiseTable(self.config['noise_size'])
model = MujocoModel(self.config['act_dim'])
algorithm = ES(model)
self.agent = MujocoAgent(algorithm, self.config)
self.latest_flat_weights = self.agent.get_flat_weights()
self.latest_obs_filter = self.obs_filter.as_serializable()
self.sample_total_episodes = 0
self.sample_total_steps = 0
self.actors_signal_input_queues = []
self.actors_output_queues = []
self.create_actors()
self.eval_rewards_stat = WindowStat(self.config['report_window_size'])
self.eval_lengths_stat = WindowStat(self.config['report_window_size'])
def create_actors(self):
""" create actors for parallel training.
"""
parl.connect(self.config['master_address'])
self.remote_count = 0
for i in range(self.config['actor_num']):
signal_queue = queue.Queue()
output_queue = queue.Queue()
self.actors_signal_input_queues.append(signal_queue)
self.actors_output_queues.append(output_queue)
self.remote_count += 1
remote_thread = threading.Thread(target=self.run_remote_sample,
args=(signal_queue, output_queue))
remote_thread.setDaemon(True)
remote_thread.start()
logger.info('All remote actors are ready, begin to learn.')
def run_remote_sample(self, signal_queue, output_queue):
""" Sample data from remote actor or get filters of remote actor.
"""
remote_actor = Actor(self.config)
while True:
info = signal_queue.get()
if info['signal'] == 'sample':
result = remote_actor.sample(self.latest_flat_weights)
output_queue.put(result)
elif info['signal'] == 'get_filter':
actor_filter = remote_actor.get_filter(flush_after=True)
output_queue.put(actor_filter)
elif info['signal'] == 'set_filter':
remote_actor.set_filter(self.latest_obs_filter)
else:
raise NotImplementedError
def step(self):
"""Run a step in ES.
1. kick off all actors to synchronize weights and sample data;
2. update parameters of the model based on sampled data.
3. update global observation filter based on local filters of all actors, and synchronize global
filter to all actors.
"""
num_episodes, num_timesteps = 0, 0
results = []
while num_episodes < self.config['min_episodes_per_batch'] or \
num_timesteps < self.config['min_steps_per_batch']:
# Send sample signal to all actors
for q in self.actors_signal_input_queues:
q.put({'signal': 'sample'})
# Collect results from all actors
for q in self.actors_output_queues:
result = q.get()
results.append(result)
# result['noisy_lengths'] is a list of lists, where the inner lists have length 2.
num_episodes += sum(
len(pair) for pair in result['noisy_lengths'])
num_timesteps += sum(
sum(pair) for pair in result['noisy_lengths'])
all_noise_indices = []
all_training_rewards = []
all_training_lengths = []
all_eval_rewards = []
all_eval_lengths = []
for result in results:
all_eval_rewards.extend(result['eval_rewards'])
all_eval_lengths.extend(result['eval_lengths'])
all_noise_indices.extend(result['noise_indices'])
all_training_rewards.extend(result['noisy_rewards'])
all_training_lengths.extend(result['noisy_lengths'])
assert len(all_eval_rewards) == len(all_eval_lengths)
assert (len(all_noise_indices) == len(all_training_rewards) ==
len(all_training_lengths))
self.sample_total_episodes += num_episodes
self.sample_total_steps += num_timesteps
eval_rewards = np.array(all_eval_rewards)
eval_lengths = np.array(all_eval_lengths)
noise_indices = np.array(all_noise_indices)
noisy_rewards = np.array(all_training_rewards)
noisy_lengths = np.array(all_training_lengths)
# normalize rewards to (-0.5, 0.5)
proc_noisy_rewards = utils.compute_centered_ranks(noisy_rewards)
noises = [
self.noise.get(index, self.agent.weights_total_size)
for index in noise_indices
]
# Update the parameters of the model.
self.agent.learn(proc_noisy_rewards, noises)
self.latest_flat_weights = self.agent.get_flat_weights()
# Update obs filter
self._update_filter()
# Store the evaluate rewards
if len(all_eval_rewards) > 0:
self.eval_rewards_stat.add(np.mean(eval_rewards))
self.eval_lengths_stat.add(np.mean(eval_lengths))
metrics = {
"episodes_this_iter": noisy_lengths.size,
"sample_total_episodes": self.sample_total_episodes,
'sample_total_steps': self.sample_total_steps,
"evaluate_rewards_mean": self.eval_rewards_stat.mean,
"evaluate_steps_mean": self.eval_lengths_stat.mean,
"timesteps_this_iter": noisy_lengths.sum(),
}
self.log_metrics(metrics)
return metrics
def _update_filter(self):
# Send get_filter signal to all actors
for q in self.actors_signal_input_queues:
q.put({'signal': 'get_filter'})
filters = []
# Collect filters from all actors and update global filter
for q in self.actors_output_queues:
actor_filter = q.get()
self.obs_filter.apply_changes(actor_filter)
# Send set_filter signal to all actors
self.latest_obs_filter = self.obs_filter.as_serializable()
for q in self.actors_signal_input_queues:
q.put({'signal': 'set_filter'})
def log_metrics(self, metrics):
logger.info(metrics)
for k, v in metrics.items():
if v is not None:
summary.add_scalar(k, v, self.sample_total_steps)
class Learner(object):
def __init__(self, config, cuda):
self.cuda = cuda
self.config = config
env = gym.make(config['env_name'])
env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')
obs_shape = env.observation_space.shape
act_dim = env.action_space.n
self.config['obs_shape'] = obs_shape
self.config['act_dim'] = act_dim
model = ActorCritic(act_dim)
if self.cuda:
model = model.cuda()
algorithm = A2C(model, config)
self.agent = Agent(algorithm, config)
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_YOU_WANT_TO_USE]` .'
else:
os.environ['CPU_NUM'] = str(1)
#========== Learner ==========
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.lr = None
self.entropy_coeff = None
self.learn_time_stat = TimeStat(100)
self.start_time = None
#========== Remote Actor ===========
self.remote_count = 0
self.sample_total_steps = 0
self.sample_data_queue = queue.Queue()
self.remote_metrics_queue = queue.Queue()
self.params_queues = []
self.create_actors()
def create_actors(self):
parl.connect(self.config['master_address'])
logger.info('Waiting for {} remote actors to connect.'.format(
self.config['actor_num']))
for i in six.moves.range(self.config['actor_num']):
params_queue = queue.Queue()
self.params_queues.append(params_queue)
self.remote_count += 1
logger.info('Remote actor count: {}'.format(self.remote_count))
remote_thread = threading.Thread(
target=self.run_remote_sample, args=(params_queue, ))
remote_thread.setDaemon(True)
remote_thread.start()
logger.info('All remote actors are ready, begin to learn.')
self.start_time = time.time()
def run_remote_sample(self, params_queue):
remote_actor = Actor(self.config)
cnt = 0
while True:
latest_params = params_queue.get()
remote_actor.set_weights(latest_params)
batch = remote_actor.sample()
self.sample_data_queue.put(batch)
cnt += 1
if cnt % self.config['get_remote_metrics_interval'] == 0:
metrics = remote_actor.get_metrics()
if metrics:
self.remote_metrics_queue.put(metrics)
def step(self):
latest_params = self.agent.get_weights()
for params_queue in self.params_queues:
params_queue.put(latest_params)
train_batch = defaultdict(list)
for i in range(self.config['actor_num']):
sample_data = self.sample_data_queue.get()
for key, value in sample_data.items():
train_batch[key].append(value)
self.sample_total_steps += len(sample_data['obs'])
for key, value in train_batch.items():
train_batch[key] = np.concatenate(value)
train_batch[key] = torch.tensor(train_batch[key]).float()
if self.cuda:
train_batch[key] = train_batch[key].cuda()
with self.learn_time_stat:
total_loss, pi_loss, vf_loss, entropy, lr, entropy_coeff = self.agent.learn(
obs=train_batch['obs'],
actions=train_batch['actions'],
advantages=train_batch['advantages'],
target_values=train_batch['target_values'],
)
self.total_loss_stat.add(total_loss.item())
self.pi_loss_stat.add(pi_loss.item())
self.vf_loss_stat.add(vf_loss.item())
self.entropy_stat.add(entropy.item())
self.lr = lr
self.entropy_coeff = entropy_coeff
def log_metrics(self):
""" Log metrics of learner and actors
"""
if self.start_time is None:
return
metrics = []
while True:
try:
metric = self.remote_metrics_queue.get_nowait()
metrics.append(metric)
except queue.Empty:
break
episode_rewards, episode_steps = [], []
for x in metrics:
episode_rewards.extend(x['episode_rewards'])
episode_steps.extend(x['episode_steps'])
max_episode_rewards, mean_episode_rewards, min_episode_rewards, \
max_episode_steps, mean_episode_steps, min_episode_steps =\
None, None, None, None, None, None
if episode_rewards:
mean_episode_rewards = np.mean(np.array(episode_rewards).flatten())
max_episode_rewards = np.max(np.array(episode_rewards).flatten())
min_episode_rewards = np.min(np.array(episode_rewards).flatten())
mean_episode_steps = np.mean(np.array(episode_steps).flatten())
max_episode_steps = np.max(np.array(episode_steps).flatten())
min_episode_steps = np.min(np.array(episode_steps).flatten())
metric = {
'Sample steps': self.sample_total_steps,
'max_episode_rewards': max_episode_rewards,
'mean_episode_rewards': mean_episode_rewards,
'min_episode_rewards': min_episode_rewards,
'max_episode_steps': max_episode_steps,
'mean_episode_steps': mean_episode_steps,
'min_episode_steps': min_episode_steps,
'total_loss': self.total_loss_stat.mean,
'pi_loss': self.pi_loss_stat.mean,
'vf_loss': self.vf_loss_stat.mean,
'entropy': self.entropy_stat.mean,
'learn_time_s': self.learn_time_stat.mean,
'elapsed_time_s': int(time.time() - self.start_time),
'lr': self.lr,
'entropy_coeff': self.entropy_coeff,
}
if metric['mean_episode_rewards'] is not None:
summary.add_scalar('train/mean_reward',
metric['mean_episode_rewards'],
self.sample_total_steps)
summary.add_scalar('train/total_loss', metric['total_loss'],
self.sample_total_steps)
summary.add_scalar('train/pi_loss', metric['pi_loss'],
self.sample_total_steps)
summary.add_scalar('train/vf_loss', metric['vf_loss'],
self.sample_total_steps)
summary.add_scalar('train/entropy', metric['entropy'],
self.sample_total_steps)
summary.add_scalar('train/learn_rate', metric['lr'],
self.sample_total_steps)
logger.info(metric)
def should_stop(self):
return self.sample_total_steps >= self.config['max_sample_steps']
class Learner(object):
def __init__(self, config):
self.config = config
# 这里创建游戏单纯是为了获取游戏动作的维度
env = retro_util.RetroEnv(game=config['env_name'],
use_restricted_actions=retro.Actions.DISCRETE,
resize_shape=config['obs_shape'],
render_preprocess=False)
obs_dim = env.observation_space.shape
action_dim = env.action_space.n
self.config['action_dim'] = action_dim
# 这里创建的模型是真正学习使用的
model = Model(action_dim)
algorithm = parl.algorithms.A3C(model, vf_loss_coeff=config['vf_loss_coeff'])
self.agent = Agent(algorithm, config, obs_dim)
# 只支持单个GPU
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_TO_USE]` .'
# 加载预训练模型
if self.config['restore_model']:
logger.info("加载预训练模型...")
self.agent.restore(self.config['model_path'])
# 记录训练的日志
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.lr = None
self.entropy_coeff = None
self.best_loss = None
self.learn_time_stat = TimeStat(100)
self.start_time = None
# ========== Remote Actor ===========
self.remote_count = 0
self.sample_data_queue = queue.Queue()
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.params_queues = []
self.create_actors()
# 开始创建指定数量的Actor，并发放的集群中
def create_actors(self):
# 连接到集群
parl.connect(self.config['master_address'])
logger.info('Waiting for {} remote actors to connect.'.format(self.config['actor_num']))
# 循环生成多个Actor线程
for i in range(self.config['actor_num']):
# 更新参数的队列
params_queue = queue.Queue()
self.params_queues.append(params_queue)
self.remote_count += 1
logger.info('Remote actor count: {}'.format(self.remote_count))
# 创建Actor的线程
remote_thread = threading.Thread(target=self.run_remote_sample, args=(params_queue,))
remote_thread.setDaemon(True)
remote_thread.start()
logger.info('All remote actors are ready, begin to learn.')
self.start_time = time.time()
# 创建Actor，并使用无限循环更新Actor的模型参数和获取游戏数据
def run_remote_sample(self, params_queue):
# 创建Actor
remote_actor = Actor(self.config)
while True:
# 获取train的模型参数
latest_params = params_queue.get()
# 设置Actor中的模型参数
remote_actor.set_weights(latest_params)
# 获取一小批的游戏数据
batch = remote_actor.sample()
# 将游戏数据添加的数据队列中
self.sample_data_queue.put(batch)
# 开始模型训练
def step(self):
"""
1. 启动所有Actor，同步参数和样本数据;
2. 收集所有Actor生成的数据;
3. 更新参数.
"""
# 获取train中模型最新的参数
latest_params = self.agent.get_weights()
# 将参数同步给没有Actor线程的参数队列
for params_queue in self.params_queues:
params_queue.put(latest_params)
train_batch = defaultdict(list)
# 获取每个Actor生成的数据
for i in range(self.config['actor_num']):
sample_data = self.sample_data_queue.get()
for key, value in sample_data.items():
train_batch[key].append(value)
# 记录训练步数
self.sample_total_steps += sample_data['obs'].shape[0]
# 将各个Actor的数据打包的训练数据
for key, value in train_batch.items():
train_batch[key] = np.concatenate(value)
# 执行一次训练
with self.learn_time_stat:
total_loss, pi_loss, vf_loss, entropy, lr, entropy_coeff = self.agent.learn(
obs_np=train_batch['obs'],
actions_np=train_batch['actions'],
advantages_np=train_batch['advantages'],
target_values_np=train_batch['target_values'])
# 记录训练数据
self.total_loss_stat.add(total_loss)
self.pi_loss_stat.add(pi_loss)
self.vf_loss_stat.add(vf_loss)
self.entropy_stat.add(entropy)
self.lr = lr
self.entropy_coeff = entropy_coeff
# 保存训练日志
def log_metrics(self):
# 避免训练还未开始的情况
if self.start_time is None:
return
# 获取最好的模型
if self.best_loss is None:
self.best_loss = self.total_loss_stat.mean
else:
if self.best_loss > self.total_loss_stat.mean:
self.best_loss = self.total_loss_stat.mean
self.save_model("model_best")
# 训练数据写入到日志中
summary.add_scalar('total_loss', self.total_loss_stat.mean, self.sample_total_steps)
summary.add_scalar('pi_loss', self.pi_loss_stat.mean, self.sample_total_steps)
summary.add_scalar('vf_loss', self.vf_loss_stat.mean, self.sample_total_steps)
summary.add_scalar('entropy', self.entropy_stat.mean, self.sample_total_steps)
summary.add_scalar('lr', self.lr, self.sample_total_steps)
summary.add_scalar('entropy_coeff', self.entropy_coeff, self.sample_total_steps)
logger.info('total_loss: {}'.format(self.total_loss_stat.mean))
# 保存模型
def save_model(self, model_name="model"):
# 避免训练还未开始的情况
if self.start_time is None:
return
save_path = os.path.join(self.config['model_path'], model_name)
if not os.path.exists(save_path):
os.makedirs(save_path)
self.agent.save(save_path)
# 检测训练步数是否达到最大步数
def should_stop(self):
return self.sample_total_steps >= self.config['max_sample_steps']
class Learner(object):
def __init__(self, config):
self.config = config
self.sample_data_queue = queue.Queue(
maxsize=config['sample_queue_max_size'])
#=========== Create Agent ==========
env = IntraBuildingEnv("config.ini")
self._mansion_attr = env._mansion.attribute
self._obs_dim = obs_dim(self._mansion_attr)
self._act_dim = act_dim(self._mansion_attr)
self.config['obs_shape'] = self._obs_dim
self.config['act_dim'] = self._act_dim
model = RLDispatcherModel(self._act_dim)
algorithm = IMPALA(model, hyperparas=config)
self.agent = ElevatorAgent(algorithm, config, self.learn_data_provider)
self.cache_params = self.agent.get_params()
self.params_lock = threading.Lock()
self.params_updated = False
self.cache_params_sent_cnt = 0
self.total_params_sync = 0
#========== Learner ==========
self.lr, self.entropy_coeff = None, None
self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.kl_stat = WindowStat(100)
self.learn_time_stat = TimeStat(100)
self.start_time = None
self.learn_thread = threading.Thread(target=self.run_learn)
self.learn_thread.setDaemon(True)
self.learn_thread.start()
#========== Remote Actor ===========
self.remote_count = 0
self.batch_buffer = []
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.remote_manager_thread = threading.Thread(
target=self.run_remote_manager)
self.remote_manager_thread.setDaemon(True)
self.remote_manager_thread.start()
self.csv_logger = CSVLogger(
os.path.join(logger.get_dir(), 'result.csv'))
from utils import Summary
self.summary = Summary('./output')
def learn_data_provider(self):
""" Data generator for fluid.layers.py_reader
"""
while True:
sample_data = self.sample_data_queue.get()
self.sample_total_steps += sample_data['obs'].shape[0]
self.batch_buffer.append(sample_data)
buffer_size = sum(
[data['obs'].shape[0] for data in self.batch_buffer])
if buffer_size >= self.config['train_batch_size']:
batch = {}
for key in self.batch_buffer[0].keys():
batch[key] = np.concatenate(
[data[key] for data in self.batch_buffer])
self.batch_buffer = []
obs_np = batch['obs'].astype('float32')
actions_np = batch['actions'].astype('int64')
behaviour_logits_np = batch['behaviour_logits'].astype(
'float32')
rewards_np = batch['rewards'].astype('float32')
dones_np = batch['dones'].astype('float32')
self.lr = self.lr_scheduler.step()
self.entropy_coeff = self.entropy_coeff_scheduler.step()
yield [
obs_np, actions_np, behaviour_logits_np, rewards_np,
dones_np, self.lr, self.entropy_coeff
]
def run_learn(self):
""" Learn loop
"""
while True:
with self.learn_time_stat:
total_loss, pi_loss, vf_loss, entropy, kl = self.agent.learn()
self.params_updated = True
self.total_loss_stat.add(total_loss)
self.pi_loss_stat.add(pi_loss)
self.vf_loss_stat.add(vf_loss)
self.entropy_stat.add(entropy)
self.kl_stat.add(kl)
def run_remote_manager(self):
""" Accept connection of new remote actor and start sampling of the remote actor.
"""
remote_manager = RemoteManager(port=self.config['server_port'])
logger.info('Waiting for remote actors connecting.')
while True:
remote_actor = remote_manager.get_remote()
self.remote_count += 1
logger.info('Remote actor count: {}'.format(self.remote_count))
if self.start_time is None:
self.start_time = time.time()
remote_thread = threading.Thread(
target=self.run_remote_sample, args=(remote_actor, ))
remote_thread.setDaemon(True)
remote_thread.start()
def run_remote_sample(self, remote_actor):
""" Sample data from remote actor and update parameters of remote actor.
"""
cnt = 0
remote_actor.set_params(self.cache_params)
while True:
batch = remote_actor.sample()
if batch:
self.sample_data_queue.put(batch)
cnt += 1
if cnt % self.config['get_remote_metrics_interval'] == 0:
metrics = remote_actor.get_metrics()
if metrics:
self.remote_metrics_queue.put(metrics)
self.params_lock.acquire()
if self.params_updated and self.cache_params_sent_cnt >= self.config[
'params_broadcast_interval']:
self.params_updated = False
self.cache_params = self.agent.get_params()
self.cache_params_sent_cnt = 0
self.cache_params_sent_cnt += 1
self.total_params_sync += 1
self.params_lock.release()
remote_actor.set_params(self.cache_params)
def log_metrics(self):
""" Log metrics of learner and actors
"""
if self.start_time is None:
return
metrics = []
while True:
try:
metric = self.remote_metrics_queue.get_nowait()
metrics.append(metric)
except queue.Empty:
break
episode_rewards, episode_steps, episode_shaping_rewards, episode_deliver_rewards, episode_wrong_deliver_rewards = [], [], [], [], []
for x in metrics:
episode_rewards.extend(x['episode_rewards'])
episode_steps.extend(x['episode_steps'])
episode_shaping_rewards.extend(x['episode_shaping_rewards'])
episode_deliver_rewards.extend(x['episode_deliver_rewards'])
episode_wrong_deliver_rewards.extend(x['episode_wrong_deliver_rewards'])
max_episode_rewards, mean_episode_rewards, min_episode_rewards, \
max_episode_steps, mean_episode_steps, min_episode_steps =\
None, None, None, None, None, None
mean_episode_shaping_rewards, mean_episode_deliver_rewards, mean_episode_wrong_deliver_rewards = \
None, None, None
if episode_rewards:
mean_episode_rewards = np.mean(np.array(episode_rewards).flatten())
max_episode_rewards = np.max(np.array(episode_rewards).flatten())
min_episode_rewards = np.min(np.array(episode_rewards).flatten())
mean_episode_steps = np.mean(np.array(episode_steps).flatten())
max_episode_steps = np.max(np.array(episode_steps).flatten())
min_episode_steps = np.min(np.array(episode_steps).flatten())
mean_episode_shaping_rewards = np.mean(np.array(episode_shaping_rewards).flatten())
mean_episode_deliver_rewards = np.mean(np.array(episode_deliver_rewards).flatten())
mean_episode_wrong_deliver_rewards = np.mean(np.array(episode_wrong_deliver_rewards).flatten())
#print(self.learn_time_stat.time_samples.items)
metric = {
'Sample steps': self.sample_total_steps,
'max_episode_rewards': max_episode_rewards,
'mean_episode_rewards': mean_episode_rewards,
'min_episode_rewards': min_episode_rewards,
'mean_episode_shaping_rewards': mean_episode_shaping_rewards,
'mean_episode_deliver_rewards': mean_episode_deliver_rewards,
'mean_episode_wrong_deliver_rewards': mean_episode_wrong_deliver_rewards,
#'max_episode_steps': max_episode_steps,
#'mean_episode_steps': mean_episode_steps,
#'min_episode_steps': min_episode_steps,
'sample_queue_size': self.sample_data_queue.qsize(),
'total_params_sync': self.total_params_sync,
'cache_params_sent_cnt': self.cache_params_sent_cnt,
'total_loss': self.total_loss_stat.mean,
'pi_loss': self.pi_loss_stat.mean,
'vf_loss': self.vf_loss_stat.mean,
'entropy': self.entropy_stat.mean,
'kl': self.kl_stat.mean,
'learn_time_s': self.learn_time_stat.mean,
'elapsed_time_s': int(time.time() - self.start_time),
'lr': self.lr,
'entropy_coeff': self.entropy_coeff,
}
logger.info(metric)
self.csv_logger.log_dict(metric)
self.summary.log_dict(metric, self.sample_total_steps)
def close(self):
self.csv_logger.close()
def __init__(self, window_size=1):
self.time_samples = WindowStat(window_size)
self._start_time = None
class Learner(object):
def __init__(self, config):
self.config = config
self.sample_data_queue = queue.Queue()
self.batch_buffer = defaultdict(list)
#=========== Create Agent ==========
env = gym.make(config['env_name'])
env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')
obs_shape = env.observation_space.shape
act_dim = env.action_space.n
self.config['obs_shape'] = obs_shape
self.config['act_dim'] = act_dim
model = AtariModel(act_dim)
algorithm = parl.algorithms.A3C(model,
vf_loss_coeff=config['vf_loss_coeff'])
self.agent = AtariAgent(
algorithm,
obs_shape=self.config['obs_shape'],
predict_thread_num=self.config['predict_thread_num'],
learn_data_provider=self.learn_data_provider)
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_YOU_WANT_TO_USE]` .'
else:
cpu_num = os.environ.get('CPU_NUM')
assert cpu_num is not None and cpu_num == '1', 'Only support training in single CPU,\
Please set environment variable: `export CPU_NUM=1`.'
#========== Learner ==========
self.lr, self.entropy_coeff = None, None
self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.learn_time_stat = TimeStat(100)
self.start_time = None
# learn thread
self.learn_thread = threading.Thread(target=self.run_learn)
self.learn_thread.setDaemon(True)
self.learn_thread.start()
self.predict_input_queue = queue.Queue()
# predict thread
self.predict_threads = []
for i in six.moves.range(self.config['predict_thread_num']):
predict_thread = threading.Thread(target=self.run_predict,
args=(i, ))
predict_thread.setDaemon(True)
predict_thread.start()
self.predict_threads.append(predict_thread)
#========== Remote Simulator ===========
self.remote_count = 0
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.create_actors()
def learn_data_provider(self):
""" Data generator for fluid.layers.py_reader
"""
B = self.config['train_batch_size']
while True:
sample_data = self.sample_data_queue.get()
self.sample_total_steps += len(sample_data['obs'])
for key in sample_data:
self.batch_buffer[key].extend(sample_data[key])
if len(self.batch_buffer['obs']) >= B:
batch = {}
for key in self.batch_buffer:
batch[key] = np.array(self.batch_buffer[key][:B])
obs_np = batch['obs'].astype('float32')
actions_np = batch['actions'].astype('int64')
advantages_np = batch['advantages'].astype('float32')
target_values_np = batch['target_values'].astype('float32')
self.lr = self.lr_scheduler.step()
self.lr = np.array(self.lr, dtype='float32')
self.entropy_coeff = self.entropy_coeff_scheduler.step()
self.entropy_coeff = np.array(self.entropy_coeff,
dtype='float32')
yield [
obs_np, actions_np, advantages_np, target_values_np,
self.lr, self.entropy_coeff
]
for key in self.batch_buffer:
self.batch_buffer[key] = self.batch_buffer[key][B:]
def run_predict(self, thread_id):
""" predict thread
"""
batch_ident = []
batch_obs = []
while True:
ident, obs = self.predict_input_queue.get()
batch_ident.append(ident)
batch_obs.append(obs)
while len(batch_obs) < self.config['max_predict_batch_size']:
try:
ident, obs = self.predict_input_queue.get_nowait()
batch_ident.append(ident)
batch_obs.append(obs)
except queue.Empty:
break
if batch_obs:
batch_obs = np.array(batch_obs)
actions, values = self.agent.sample(batch_obs, thread_id)
for i, ident in enumerate(batch_ident):
self.predict_output_queues[ident].put(
(actions[i], values[i]))
batch_ident = []
batch_obs = []
def run_learn(self):
""" Learn loop
"""
while True:
with self.learn_time_stat:
total_loss, pi_loss, vf_loss, entropy = self.agent.learn()
self.total_loss_stat.add(total_loss)
self.pi_loss_stat.add(pi_loss)
self.vf_loss_stat.add(vf_loss)
self.entropy_stat.add(entropy)
def create_actors(self):
""" Connect to the cluster and start sampling of the remote actor.
"""
parl.connect(self.config['master_address'])
logger.info('Waiting for {} remote actors to connect.'.format(
self.config['actor_num']))
ident = 0
self.predict_output_queues = []
for i in six.moves.range(self.config['actor_num']):
self.remote_count += 1
logger.info('Remote simulator count: {}'.format(self.remote_count))
if self.start_time is None:
self.start_time = time.time()
q = queue.Queue()
self.predict_output_queues.append(q)
remote_thread = threading.Thread(target=self.run_remote_sample,
args=(ident, ))
remote_thread.setDaemon(True)
remote_thread.start()
ident += 1
def run_remote_sample(self, ident):
""" Interacts with remote simulator.
"""
remote_actor = Actor(self.config)
mem = defaultdict(list)
obs = remote_actor.reset()
while True:
self.predict_input_queue.put((ident, obs))
action, value = self.predict_output_queues[ident].get()
next_obs, reward, done = remote_actor.step(action)
mem['obs'].append(obs)
mem['actions'].append(action)
mem['rewards'].append(reward)
mem['values'].append(value)
if done:
next_value = 0
advantages = calc_gae(mem['rewards'], mem['values'],
next_value, self.config['gamma'],
self.config['lambda'])
target_values = advantages + mem['values']
self.sample_data_queue.put({
'obs': mem['obs'],
'actions': mem['actions'],
'advantages': advantages,
'target_values': target_values
})
mem = defaultdict(list)
next_obs = remote_actor.reset()
elif len(mem['obs']) == self.config['t_max'] + 1:
next_value = mem['values'][-1]
advantages = calc_gae(mem['rewards'][:-1], mem['values'][:-1],
next_value, self.config['gamma'],
self.config['lambda'])
target_values = advantages + mem['values'][:-1]
self.sample_data_queue.put({
'obs': mem['obs'][:-1],
'actions': mem['actions'][:-1],
'advantages': advantages,
'target_values': target_values
})
for key in mem:
mem[key] = [mem[key][-1]]
obs = next_obs
if done:
metrics = remote_actor.get_metrics()
if metrics:
self.remote_metrics_queue.put(metrics)
def log_metrics(self):
""" Log metrics of learner and simulators
"""
if self.start_time is None:
return
metrics = []
while True:
try:
metric = self.remote_metrics_queue.get_nowait()
metrics.append(metric)
except queue.Empty:
break
episode_rewards, episode_steps = [], []
for x in metrics:
episode_rewards.extend(x['episode_rewards'])
episode_steps.extend(x['episode_steps'])
max_episode_rewards, mean_episode_rewards, min_episode_rewards, \
max_episode_steps, mean_episode_steps, min_episode_steps =\
None, None, None, None, None, None
if episode_rewards:
mean_episode_rewards = np.mean(np.array(episode_rewards).flatten())
max_episode_rewards = np.max(np.array(episode_rewards).flatten())
min_episode_rewards = np.min(np.array(episode_rewards).flatten())
mean_episode_steps = np.mean(np.array(episode_steps).flatten())
max_episode_steps = np.max(np.array(episode_steps).flatten())
min_episode_steps = np.min(np.array(episode_steps).flatten())
metric = {
'Sample steps': self.sample_total_steps,
'max_episode_rewards': max_episode_rewards,
'mean_episode_rewards': mean_episode_rewards,
'min_episode_rewards': min_episode_rewards,
'max_episode_steps': max_episode_steps,
'mean_episode_steps': mean_episode_steps,
'min_episode_steps': min_episode_steps,
'total_loss': self.total_loss_stat.mean,
'pi_loss': self.pi_loss_stat.mean,
'vf_loss': self.vf_loss_stat.mean,
'entropy': self.entropy_stat.mean,
'learn_time_s': self.learn_time_stat.mean,
'elapsed_time_s': int(time.time() - self.start_time),
'lr': self.lr,
'entropy_coeff': self.entropy_coeff,
}
for key, value in metric.items():
if value is not None:
summary.add_scalar(key, value, self.sample_total_steps)
logger.info(metric)
def __init__(self, config):
self.config = config
self.sample_data_queue = queue.Queue()
self.batch_buffer = defaultdict(list)
#=========== Create Agent ==========
env = gym.make(config['env_name'])
env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')
obs_shape = env.observation_space.shape
act_dim = env.action_space.n
self.config['obs_shape'] = obs_shape
self.config['act_dim'] = act_dim
model = AtariModel(act_dim)
algorithm = parl.algorithms.A3C(model,
vf_loss_coeff=config['vf_loss_coeff'])
self.agent = AtariAgent(
algorithm,
obs_shape=self.config['obs_shape'],
predict_thread_num=self.config['predict_thread_num'],
learn_data_provider=self.learn_data_provider)
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_YOU_WANT_TO_USE]` .'
else:
cpu_num = os.environ.get('CPU_NUM')
assert cpu_num is not None and cpu_num == '1', 'Only support training in single CPU,\
Please set environment variable: `export CPU_NUM=1`.'
#========== Learner ==========
self.lr, self.entropy_coeff = None, None
self.lr_scheduler = PiecewiseScheduler(config['lr_scheduler'])
self.entropy_coeff_scheduler = PiecewiseScheduler(
config['entropy_coeff_scheduler'])
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.learn_time_stat = TimeStat(100)
self.start_time = None
# learn thread
self.learn_thread = threading.Thread(target=self.run_learn)
self.learn_thread.setDaemon(True)
self.learn_thread.start()
self.predict_input_queue = queue.Queue()
# predict thread
self.predict_threads = []
for i in six.moves.range(self.config['predict_thread_num']):
predict_thread = threading.Thread(target=self.run_predict,
args=(i, ))
predict_thread.setDaemon(True)
predict_thread.start()
self.predict_threads.append(predict_thread)
#========== Remote Simulator ===========
self.remote_count = 0
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.create_actors()
class Learner(object):
def __init__(self, config):
self.config = config
#=========== Create Agent ==========
env = gym.make(config['env_name'])
env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')
obs_shape = env.observation_space.shape
act_dim = env.action_space.n
self.config['obs_shape'] = obs_shape
self.config['act_dim'] = act_dim
model = AtariModel(act_dim)
algorithm = parl.algorithms.A3C(model,
vf_loss_coeff=config['vf_loss_coeff'])
self.agent = AtariAgent(algorithm, config)
if machine_info.is_gpu_available():
assert get_gpu_count() == 1, 'Only support training in single GPU,\
Please set environment variable: `export CUDA_VISIBLE_DEVICES=[GPU_ID_TO_USE]` .'
#========== Learner ==========
self.total_loss_stat = WindowStat(100)
self.pi_loss_stat = WindowStat(100)
self.vf_loss_stat = WindowStat(100)
self.entropy_stat = WindowStat(100)
self.lr = None
self.entropy_coeff = None
self.learn_time_stat = TimeStat(100)
self.start_time = None
#========== Remote Actor ===========
self.remote_count = 0
self.sample_data_queue = queue.Queue()
self.remote_metrics_queue = queue.Queue()
self.sample_total_steps = 0
self.params_queues = []
self.create_actors()
def create_actors(self):
""" Connect to the cluster and start sampling of the remote actor.
"""
parl.connect(self.config['master_address'])
logger.info('Waiting for {} remote actors to connect.'.format(
self.config['actor_num']))
for i in six.moves.range(self.config['actor_num']):
params_queue = queue.Queue()
self.params_queues.append(params_queue)
self.remote_count += 1
logger.info('Remote actor count: {}'.format(self.remote_count))
remote_thread = threading.Thread(target=self.run_remote_sample,
args=(params_queue, ))
remote_thread.setDaemon(True)
remote_thread.start()
logger.info('All remote actors are ready, begin to learn.')
self.start_time = time.time()
def run_remote_sample(self, params_queue):
""" Sample data from remote actor and update parameters of remote actor.
"""
remote_actor = Actor(self.config)
cnt = 0
while True:
latest_params = params_queue.get()
remote_actor.set_weights(latest_params)
batch = remote_actor.sample()
self.sample_data_queue.put(batch)
cnt += 1
if cnt % self.config['get_remote_metrics_interval'] == 0:
metrics = remote_actor.get_metrics()
if metrics:
self.remote_metrics_queue.put(metrics)
def step(self):
"""
1. kick off all actors to synchronize parameters and sample data;
2. collect sample data of all actors;
3. update parameters.
"""
latest_params = self.agent.get_weights()
for params_queue in self.params_queues:
params_queue.put(latest_params)
train_batch = defaultdict(list)
for i in range(self.config['actor_num']):
sample_data = self.sample_data_queue.get()
for key, value in sample_data.items():
train_batch[key].append(value)
self.sample_total_steps += sample_data['obs'].shape[0]
for key, value in train_batch.items():
train_batch[key] = np.concatenate(value)
with self.learn_time_stat:
total_loss, pi_loss, vf_loss, entropy, lr, entropy_coeff = self.agent.learn(
obs_np=train_batch['obs'],
actions_np=train_batch['actions'],
advantages_np=train_batch['advantages'],
target_values_np=train_batch['target_values'])
self.total_loss_stat.add(total_loss)
self.pi_loss_stat.add(pi_loss)
self.vf_loss_stat.add(vf_loss)
self.entropy_stat.add(entropy)
self.lr = lr
self.entropy_coeff = entropy_coeff
def log_metrics(self):
""" Log metrics of learner and actors
"""
if self.start_time is None:
return
metrics = []
while True:
try:
metric = self.remote_metrics_queue.get_nowait()
metrics.append(metric)
except queue.Empty:
break
episode_rewards, episode_steps = [], []
for x in metrics:
episode_rewards.extend(x['episode_rewards'])
episode_steps.extend(x['episode_steps'])
max_episode_rewards, mean_episode_rewards, min_episode_rewards, \
max_episode_steps, mean_episode_steps, min_episode_steps =\
None, None, None, None, None, None
if episode_rewards:
mean_episode_rewards = np.mean(np.array(episode_rewards).flatten())
max_episode_rewards = np.max(np.array(episode_rewards).flatten())
min_episode_rewards = np.min(np.array(episode_rewards).flatten())
mean_episode_steps = np.mean(np.array(episode_steps).flatten())
max_episode_steps = np.max(np.array(episode_steps).flatten())
min_episode_steps = np.min(np.array(episode_steps).flatten())
metric = {
'sample_steps': self.sample_total_steps,
'max_episode_rewards': max_episode_rewards,
'mean_episode_rewards': mean_episode_rewards,
'min_episode_rewards': min_episode_rewards,
'max_episode_steps': max_episode_steps,
'mean_episode_steps': mean_episode_steps,
'min_episode_steps': min_episode_steps,
'total_loss': self.total_loss_stat.mean,
'pi_loss': self.pi_loss_stat.mean,
'vf_loss': self.vf_loss_stat.mean,
'entropy': self.entropy_stat.mean,
'learn_time_s': self.learn_time_stat.mean,
'elapsed_time_s': int(time.time() - self.start_time),
'lr': self.lr,
'entropy_coeff': self.entropy_coeff,
}
for key, value in metric.items():
if value is not None:
summary.add_scalar(key, value, self.sample_total_steps)
logger.info(metric)
def should_stop(self):
return self.sample_total_steps >= self.config['max_sample_steps']
