class Actor(object):
def __init__(self, config):
self.config = config
self.envs = []
for _ in range(config['env_num']):
env = gym.make(config['env_name'])
env.seed(ENV_SEED)
env = MonitorEnv(env)
env = ClipRewardEnv(env)
env = StateStack(env, k=4)
self.envs.append(env)
# env = gym.make(config['env_name'])
# obs_shape = env.observation_space.shape
self.vector_env = VectorEnv(self.envs)
self.obs_batch = self.vector_env.reset()
obs_dim = self.envs[0].observation_space.shape
act_dim = self.envs[0].action_space.shape[0]
max_action = float(self.envs[0].action_space.high[0])
# obs_shape = env.observation_space.shape
# act_dim = env.action_space.n
model = MujocoModel(act_dim)
algorithm = DVtrace(
model,
max_action,
sample_batch_steps=self.config['sample_batch_steps'],
gamma=self.config['gamma'],
vf_loss_coeff=self.config['vf_loss_coeff'],
clip_rho_threshold=self.config['clip_rho_threshold'],
clip_pg_rho_threshold=self.config['clip_pg_rho_threshold'])
self.agent = AtariAgent(algorithm, obs_dim, act_dim)
def sample(self):
env_sample_data = {}
for env_id in range(self.config['env_num']):
env_sample_data[env_id] = defaultdict(list)
for i in range(self.config['sample_batch_steps']):
actions, mean, std = self.agent.sample(self.obs_batch)
next_obs_batch, reward_batch, done_batch, info_batch = self.vector_env.step(actions)
for env_id in range(self.config['env_num']):
env_sample_data[env_id]['obs'].append(self.obs_batch[env_id])
env_sample_data[env_id]['actions'].append(actions[env_id])
env_sample_data[env_id]['mean'].append(mean[env_id])
env_sample_data[env_id]['std'].append(std[env_id])
env_sample_data[env_id]['rewards'].append(reward_batch[env_id])
env_sample_data[env_id]['dones'].append(done_batch[env_id])
self.obs_batch = next_obs_batch
# Merge data of envs
sample_data = defaultdict(list)
for env_id in range(self.config['env_num']):
for data_name in [
'obs', 'actions', 'mean', 'std', 'rewards', 'dones'
]:
sample_data[data_name].extend(
env_sample_data[env_id][data_name])
# size of sample_data: env_num * sample_batch_steps
for key in sample_data:
sample_data[key] = np.stack(sample_data[key])
return sample_data
def get_metrics(self):
metrics = defaultdict(list)
for env in self.envs:
monitor = get_wrapper_by_cls(env, MonitorEnv)
if monitor is not None:
for episode_rewards, episode_steps in monitor.next_episode_results(
):
metrics['episode_rewards'].append(episode_rewards)
metrics['episode_steps'].append(episode_steps)
return metrics
def set_weights(self, weights):
self.agent.set_weights(weights)
class Actor(object):
def __init__(self, config):
self.config = config
self.envs = []
for _ in range(config['env_num']):
env = gym.make(config['env_name'])
env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')
self.envs.append(env)
self.vector_env = VectorEnv(self.envs)
self.obs_batch = self.vector_env.reset()
obs_shape = env.observation_space.shape
act_dim = env.action_space.n
model = AtariModel(act_dim)
algorithm = parl.algorithms.IMPALA(
model,
sample_batch_steps=self.config['sample_batch_steps'],
gamma=self.config['gamma'],
vf_loss_coeff=self.config['vf_loss_coeff'],
clip_rho_threshold=self.config['clip_rho_threshold'],
clip_pg_rho_threshold=self.config['clip_pg_rho_threshold'])
self.agent = AtariAgent(algorithm, obs_shape, act_dim)
def sample(self):
env_sample_data = {}
for env_id in range(self.config['env_num']):
env_sample_data[env_id] = defaultdict(list)
for i in range(self.config['sample_batch_steps']):
actions, behaviour_logits = self.agent.sample(
np.stack(self.obs_batch))
next_obs_batch, reward_batch, done_batch, info_batch = \
self.vector_env.step(actions)
for env_id in range(self.config['env_num']):
env_sample_data[env_id]['obs'].append(self.obs_batch[env_id])
env_sample_data[env_id]['actions'].append(actions[env_id])
env_sample_data[env_id]['behaviour_logits'].append(
behaviour_logits[env_id])
env_sample_data[env_id]['rewards'].append(reward_batch[env_id])
env_sample_data[env_id]['dones'].append(done_batch[env_id])
self.obs_batch = next_obs_batch
# Merge data of envs
sample_data = defaultdict(list)
for env_id in range(self.config['env_num']):
for data_name in [
'obs', 'actions', 'behaviour_logits', 'rewards', 'dones'
]:
sample_data[data_name].extend(
env_sample_data[env_id][data_name])
# size of sample_data: env_num * sample_batch_steps
for key in sample_data:
sample_data[key] = np.stack(sample_data[key])
return sample_data
def get_metrics(self):
metrics = defaultdict(list)
for env in self.envs:
monitor = get_wrapper_by_cls(env, MonitorEnv)
if monitor is not None:
for episode_rewards, episode_steps in monitor.next_episode_results(
):
metrics['episode_rewards'].append(episode_rewards)
metrics['episode_steps'].append(episode_steps)
return metrics
def set_weights(self, weights):
self.agent.set_weights(weights)
class Actor(object):
def __init__(self, config):
self.config = config
self.envs = []
for _ in range(config['env_num']):
env = gym.make(config['env_name'])
env = wrap_deepmind(env, dim=config['env_dim'], obs_format='NCHW')
self.envs.append(env)
self.vector_env = VectorEnv(self.envs)
self.obs_batch = self.vector_env.reset()
obs_shape = env.observation_space.shape
act_dim = env.action_space.n
self.config['obs_shape'] = obs_shape
self.config['act_dim'] = act_dim
model = AtariModel(act_dim)
algorithm = parl.algorithms.A3C(model,
vf_loss_coeff=config['vf_loss_coeff'])
self.agent = AtariAgent(algorithm, config)
def sample(self):
sample_data = defaultdict(list)
env_sample_data = {}
for env_id in range(self.config['env_num']):
env_sample_data[env_id] = defaultdict(list)
for i in range(self.config['sample_batch_steps']):
actions_batch, values_batch = self.agent.sample(
np.stack(self.obs_batch))
next_obs_batch, reward_batch, done_batch, info_batch = \
self.vector_env.step(actions_batch)
for env_id in range(self.config['env_num']):
env_sample_data[env_id]['obs'].append(self.obs_batch[env_id])
env_sample_data[env_id]['actions'].append(
actions_batch[env_id])
env_sample_data[env_id]['rewards'].append(reward_batch[env_id])
env_sample_data[env_id]['dones'].append(done_batch[env_id])
env_sample_data[env_id]['values'].append(values_batch[env_id])
# Calculate advantages when the episode is done or reach max sample steps.
if done_batch[
env_id] or i == self.config['sample_batch_steps'] - 1:
next_value = 0
if not done_batch[env_id]:
next_obs = np.expand_dims(next_obs_batch[env_id], 0)
next_value = self.agent.value(next_obs)
values = env_sample_data[env_id]['values']
rewards = env_sample_data[env_id]['rewards']
advantages = calc_gae(rewards, values, next_value,
self.config['gamma'],
self.config['lambda'])
target_values = advantages + values
sample_data['obs'].extend(env_sample_data[env_id]['obs'])
sample_data['actions'].extend(
env_sample_data[env_id]['actions'])
sample_data['advantages'].extend(advantages)
sample_data['target_values'].extend(target_values)
env_sample_data[env_id] = defaultdict(list)
self.obs_batch = next_obs_batch
# size of sample_data: env_num * sample_batch_steps
for key in sample_data:
sample_data[key] = np.stack(sample_data[key])
return sample_data
def get_metrics(self):
metrics = defaultdict(list)
for env in self.envs:
monitor = get_wrapper_by_cls(env, MonitorEnv)
if monitor is not None:
for episode_rewards, episode_steps in monitor.next_episode_results(
):
metrics['episode_rewards'].append(episode_rewards)
metrics['episode_steps'].append(episode_steps)
return metrics
def set_weights(self, params):
self.agent.set_weights(params)