def __init__(self, base_name, config):
a2c_common.DiscreteA2CBase.__init__(self, base_name, config)
obs_shape = torch_ext.shape_whc_to_cwh(self.obs_shape)
config = {
'actions_num': self.actions_num,
'input_shape': obs_shape,
'num_seqs': self.num_actors * self.num_agents,
'value_size': self.env_info.get('value_size', 1)
}
self.model = self.network.build(config)
self.model.to(self.ppo_device)
self.init_rnn_from_model(self.model)
self.last_lr = float(self.last_lr)
self.optimizer = optim.Adam(self.model.parameters(),
float(self.last_lr),
eps=1e-08,
weight_decay=self.weight_decay)
if self.normalize_input:
self.running_mean_std = RunningMeanStd(obs_shape).to(
self.ppo_device)
if self.has_central_value:
cv_config = {
'state_shape': torch_ext.shape_whc_to_cwh(self.state_shape),
'value_size': self.value_size,
'ppo_device': self.ppo_device,
'num_agents': self.num_agents,
'num_steps': self.steps_num,
'num_actors': self.num_actors,
'num_actions': self.actions_num,
'seq_len': self.seq_len,
'model': self.central_value_config['network'],
'config': self.central_value_config,
'writter': self.writer,
'multi_gpu': self.multi_gpu
}
self.central_value_net = central_value.CentralValueTrain(
**cv_config).to(self.ppo_device)
self.use_experimental_cv = self.config.get('use_experimental_cv',
False)
self.dataset = datasets.PPODataset(self.batch_size,
self.minibatch_size,
self.is_discrete, self.is_rnn,
self.ppo_device, self.seq_len)
self.algo_observer.after_init(self)
def __init__(self, config):
BasePlayer.__init__(self, config)
self.network = config['network']
if type(self.action_space) is gym.spaces.Discrete:
self.actions_num = self.action_space.n
self.is_multi_discrete = False
if type(self.action_space) is gym.spaces.Tuple:
self.actions_num = [action.n for action in self.action_space]
self.is_multi_discrete = True
self.mask = [False]
self.normalize_input = self.config['normalize_input']
obs_shape = torch_ext.shape_whc_to_cwh(self.state_shape)
config = {
'actions_num': self.actions_num,
'input_shape': obs_shape,
'num_seqs': self.num_agents,
'value_size': self.value_size
}
self.model = self.network.build(config)
self.model.to(self.device)
self.model.eval()
self.is_rnn = self.model.is_rnn()
if self.normalize_input:
self.running_mean_std = RunningMeanStd(obs_shape).to(self.device)
self.running_mean_std.eval()
def __init__(self, config):
BasePlayer.__init__(self, config)
self.network = config['network']
self.actions_num = self.action_space.shape[0]
self.action_range = [
float(self.env_info['action_space'].low.min()),
float(self.env_info['action_space'].high.max())
]
obs_shape = torch_ext.shape_whc_to_cwh(self.state_shape)
self.normalize_input = False
config = {
'obs_dim': self.env_info["observation_space"].shape[0],
'action_dim': self.env_info["action_space"].shape[0],
'actions_num': self.actions_num,
'input_shape': obs_shape
}
self.model = self.network.build(config)
self.model.to(self.device)
self.model.eval()
self.is_rnn = self.model.is_rnn()
def __init__(self, config):
BasePlayer.__init__(self, config)
self.network = config['network']
self.actions_num = self.action_space.shape[0]
self.actions_low = torch.from_numpy(
self.action_space.low.copy()).float().to(self.device)
self.actions_high = torch.from_numpy(
self.action_space.high.copy()).float().to(self.device)
self.mask = [False]
self.normalize_input = self.config['normalize_input']
obs_shape = torch_ext.shape_whc_to_cwh(self.state_shape)
config = {
'actions_num': self.actions_num,
'input_shape': obs_shape,
'num_seqs': self.num_agents
}
self.model = self.network.build(config)
self.model.to(self.device)
self.model.eval()
self.is_rnn = self.model.is_rnn()
if self.normalize_input:
self.running_mean_std = RunningMeanStd(obs_shape).to(self.device)
self.running_mean_std.eval()
def __init__(self, params, **kwargs):
actions_num = kwargs.pop('actions_num')
input_shape = kwargs.pop('input_shape')
input_shape = torch_ext.shape_whc_to_cwh(input_shape)
self.num_seqs = num_seqs = kwargs.pop('num_seqs', 1)
self.value_size = kwargs.pop('value_size', 1)
NetworkBuilder.BaseNetwork.__init__(self, **kwargs)
self.load(params)
self.cnn = self._build_impala(input_shape, self.conv_depths)
mlp_input_shape = self._calc_input_size(input_shape, self.cnn)
in_mlp_shape = mlp_input_shape
if len(self.units) == 0:
out_size = mlp_input_shape
else:
out_size = self.units[-1]
if self.has_rnn:
if not self.is_rnn_before_mlp:
rnn_in_size = out_size
out_size = self.rnn_units
else:
rnn_in_size = in_mlp_shape
in_mlp_shape = self.rnn_units
self.rnn = self._build_rnn(self.rnn_name, rnn_in_size,
self.rnn_units, self.rnn_layers)
#self.layer_norm = torch.nn.LayerNorm(self.rnn_units)
mlp_args = {
'input_size': in_mlp_shape,
'units': self.units,
'activation': self.activation,
'norm_func_name': self.normalization,
'dense_func': torch.nn.Linear
}
self.mlp = self._build_mlp(**mlp_args)
self.value = torch.nn.Linear(out_size, self.value_size)
self.value_act = self.activations_factory.create(
self.value_activation)
self.flatten_act = self.activations_factory.create(self.activation)
if self.is_discrete:
self.logits = torch.nn.Linear(out_size, actions_num)
if self.is_continuous:
self.mu = torch.nn.Linear(out_size, actions_num)
self.mu_act = self.activations_factory.create(
self.space_config['mu_activation'])
mu_init = self.init_factory.create(
**self.space_config['mu_init'])
self.sigma_act = self.activations_factory.create(
self.space_config['sigma_activation'])
sigma_init = self.init_factory.create(
**self.space_config['sigma_init'])
if self.space_config['fixed_sigma']:
self.sigma = nn.Parameter(torch.zeros(actions_num,
requires_grad=True,
dtype=torch.float32),
requires_grad=True)
else:
self.sigma = torch.nn.Linear(out_size, actions_num)
mlp_init = self.init_factory.create(**self.initializer)
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out')
#nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain('elu'))
for m in self.mlp:
if isinstance(m, nn.Linear):
mlp_init(m.weight)
if self.is_discrete:
mlp_init(self.logits.weight)
if self.is_continuous:
mu_init(self.mu.weight)
if self.space_config['fixed_sigma']:
sigma_init(self.sigma)
else:
sigma_init(self.sigma.weight)
mlp_init(self.value.weight)
def __init__(self, params, **kwargs):
actions_num = kwargs.pop('actions_num')
input_shape = kwargs.pop('input_shape')
self.value_size = kwargs.pop('value_size', 1)
self.num_seqs = num_seqs = kwargs.pop('num_seqs', 1)
NetworkBuilder.BaseNetwork.__init__(self)
self.load(params)
self.actor_cnn = nn.Sequential()
self.critic_cnn = nn.Sequential()
self.actor_mlp = nn.Sequential()
self.critic_mlp = nn.Sequential()
if self.has_cnn:
input_shape = torch_ext.shape_whc_to_cwh(input_shape)
cnn_args = {
'ctype': self.cnn['type'],
'input_shape': input_shape,
'convs': self.cnn['convs'],
'activation': self.cnn['activation'],
'norm_func_name': self.normalization,
}
self.actor_cnn = self._build_conv(**cnn_args)
if self.separate:
self.critic_cnn = self._build_conv(**cnn_args)
mlp_input_shape = self._calc_input_size(input_shape,
self.actor_cnn)
if self.use_joint_obs_actions:
use_embedding = self.joint_obs_actions_config['embedding']
emb_size = self.joint_obs_actions_config['embedding_scale']
num_agents = kwargs.pop('num_agents')
mlp_out = mlp_input_shape // self.joint_obs_actions_config[
'mlp_scale']
self.joint_actions = torch_ext.DiscreteActionsEncoder(
actions_num, mlp_out, emb_size, num_agents, use_embedding)
mlp_input_shape = mlp_input_shape + mlp_out
in_mlp_shape = mlp_input_shape
if len(self.units) == 0:
out_size = mlp_input_shape
else:
out_size = self.units[-1]
if self.has_rnn:
if not self.is_rnn_before_mlp:
rnn_in_size = out_size
out_size = self.rnn_units
if self.rnn_concat_input:
rnn_in_size += in_mlp_shape
else:
rnn_in_size = in_mlp_shape
in_mlp_shape = self.rnn_units
if self.separate:
self.a_rnn = self._build_rnn(self.rnn_name, rnn_in_size,
self.rnn_units,
self.rnn_layers)
self.c_rnn = self._build_rnn(self.rnn_name, rnn_in_size,
self.rnn_units,
self.rnn_layers)
if self.rnn_ln:
self.a_layer_norm = torch.nn.LayerNorm(self.rnn_units)
self.c_layer_norm = torch.nn.LayerNorm(self.rnn_units)
else:
self.rnn = self._build_rnn(self.rnn_name, rnn_in_size,
self.rnn_units, self.rnn_layers)
if self.rnn_ln:
self.layer_norm = torch.nn.LayerNorm(self.rnn_units)
mlp_args = {
'input_size': in_mlp_shape,
'units': self.units,
'activation': self.activation,
'norm_func_name': self.normalization,
'dense_func': torch.nn.Linear,
'd2rl': self.is_d2rl,
'norm_only_first_layer': self.norm_only_first_layer
}
self.actor_mlp = self._build_mlp(**mlp_args)
if self.separate:
self.critic_mlp = self._build_mlp(**mlp_args)
self.value = torch.nn.Linear(out_size, self.value_size)
self.value_act = self.activations_factory.create(
self.value_activation)
if self.is_discrete:
self.logits = torch.nn.Linear(out_size, actions_num)
'''
for multidiscrete actions num is a tuple
'''
if self.is_multi_discrete:
self.logits = torch.nn.ModuleList(
[torch.nn.Linear(out_size, num) for num in actions_num])
if self.is_continuous:
self.mu = torch.nn.Linear(out_size, actions_num)
self.mu_act = self.activations_factory.create(
self.space_config['mu_activation'])
mu_init = self.init_factory.create(
**self.space_config['mu_init'])
self.sigma_act = self.activations_factory.create(
self.space_config['sigma_activation'])
sigma_init = self.init_factory.create(
**self.space_config['sigma_init'])
if self.space_config['fixed_sigma']:
self.sigma = nn.Parameter(torch.zeros(actions_num,
requires_grad=True,
dtype=torch.float32),
requires_grad=True)
else:
self.sigma = torch.nn.Linear(out_size, actions_num)
mlp_init = self.init_factory.create(**self.initializer)
if self.has_cnn:
cnn_init = self.init_factory.create(**self.cnn['initializer'])
for m in self.modules():
if isinstance(m, nn.Conv2d) or isinstance(m, nn.Conv1d):
cnn_init(m.weight)
if getattr(m, "bias", None) is not None:
torch.nn.init.zeros_(m.bias)
if isinstance(m, nn.Linear):
mlp_init(m.weight)
if getattr(m, "bias", None) is not None:
torch.nn.init.zeros_(m.bias)
if self.is_continuous:
mu_init(self.mu.weight)
if self.space_config['fixed_sigma']:
sigma_init(self.sigma)
else:
sigma_init(self.sigma.weight)
def __init__(self, base_name, params):
self.config = config = params['config']
print(config)
# TODO: Get obs shape and self.network
self.load_networks(params)
self.base_init(base_name, config)
self.num_seed_steps = config["num_seed_steps"]
self.gamma = config["gamma"]
self.critic_tau = config["critic_tau"]
self.batch_size = config["batch_size"]
self.init_alpha = config["init_alpha"]
self.learnable_temperature = config["learnable_temperature"]
self.replay_buffer_size = config["replay_buffer_size"]
self.num_steps_per_episode = config.get("num_steps_per_episode", 1)
self.normalize_input = config.get("normalize_input", False)
self.max_env_steps = config.get(
"max_env_steps",
1000) # temporary, in future we will use other approach
print(self.batch_size, self.num_actors, self.num_agents)
self.num_frames_per_epoch = self.num_actors * self.num_steps_per_episode
self.log_alpha = torch.tensor(np.log(self.init_alpha)).float().to(
self.sac_device)
self.log_alpha.requires_grad = True
action_space = self.env_info['action_space']
self.actions_num = action_space.shape[0]
self.action_range = [
float(self.env_info['action_space'].low.min()),
float(self.env_info['action_space'].high.max())
]
obs_shape = torch_ext.shape_whc_to_cwh(self.obs_shape)
net_config = {
'obs_dim': self.env_info["observation_space"].shape[0],
'action_dim': self.env_info["action_space"].shape[0],
'actions_num': self.actions_num,
'input_shape': obs_shape,
'normalize_input': self.nnormalize_input
}
self.model = self.network.build(net_config)
self.model.to(self.sac_device)
print("Number of Agents", self.num_actors, "Batch Size",
self.batch_size)
self.actor_optimizer = torch.optim.Adam(
self.model.sac_network.actor.parameters(),
lr=self.config['actor_lr'],
betas=self.config.get("actor_betas", [0.9, 0.999]))
self.critic_optimizer = torch.optim.Adam(
self.model.sac_network.critic.parameters(),
lr=self.config["critic_lr"],
betas=self.config.get("critic_betas", [0.9, 0.999]))
self.log_alpha_optimizer = torch.optim.Adam([self.log_alpha],
lr=self.config["alpha_lr"],
betas=self.config.get(
"alphas_betas",
[0.9, 0.999]))
self.replay_buffer = experience.VectorizedReplayBuffer(
self.env_info['observation_space'].shape,
self.env_info['action_space'].shape, self.replay_buffer_size,
self.sac_device)
self.target_entropy_coef = config.get("target_entropy_coef", 0.5)
self.target_entropy = self.target_entropy_coef * -self.env_info[
'action_space'].shape[0]
print("Target entropy", self.target_entropy)
self.step = 0
self.algo_observer = config['features']['observer']
# TODO: Is there a better way to get the maximum number of episodes?
self.max_episodes = torch.ones(
self.num_actors,
device=self.sac_device) * self.num_steps_per_episode
def __init__(self, base_name, config):
print(config)
# TODO: Get obs shape and self.network
self.base_init(base_name, config)
self.num_seed_steps = config["num_seed_steps"]
self.discount = config["discount"]
self.critic_tau = config["critic_tau"]
self.actor_update_frequency = config["actor_update_frequency"]
self.critic_target_update_frequency = config[
"critic_target_update_frequency"]
self.batch_size = config["batch_size"]
self.init_temperature = config["init_temperature"]
self.learnable_temperature = config["learnable_temperature"]
self.replay_buffer_size = config["replay_buffer_size"]
self.num_steps_per_episode = config.get("num_steps_per_episode", 500)
print(self.batch_size, self.num_actors, self.num_agents)
self.num_frames_per_epoch = self.num_actors * self.num_steps_per_episode
self.log_alpha = torch.tensor(np.log(self.init_temperature)).to(
self.sac_device)
self.log_alpha.requires_grad = True
action_space = self.env_info['action_space']
self.actions_num = action_space.shape[0]
self.action_range = [
float(self.env_info['action_space'].low.min()),
float(self.env_info['action_space'].high.max())
]
obs_shape = torch_ext.shape_whc_to_cwh(self.obs_shape)
config = {
'obs_dim': self.env_info["observation_space"].shape[0],
'action_dim': self.env_info["action_space"].shape[0],
'actions_num': self.actions_num,
'input_shape': obs_shape
}
self.model = self.network.build(config)
self.model.to(self.sac_device)
print("Number of Agents", self.num_actors, "Batch Size",
self.batch_size)
self.actor_optimizer = torch.optim.Adam(
self.model.sac_network.actor.parameters(),
lr=self.config['actor_lr'],
betas=self.config["actor_betas"])
self.critic_optimizer = torch.optim.Adam(
self.model.sac_network.critic.parameters(),
lr=self.config["critic_lr"],
betas=self.config["critic_betas"])
self.log_alpha_optimizer = torch.optim.Adam(
[self.log_alpha],
lr=self.config["alpha_lr"],
betas=self.config["alpha_betas"])
self.replay_buffer = experience.VectorizedReplayBuffer(
self.env_info['observation_space'].shape,
self.env_info['action_space'].shape, self.replay_buffer_size,
self.sac_device)
self.target_entropy = -self.env_info['action_space'].shape[0]
# self.algo_observer.after_init(self)
# TODO: Algo_Observer?
self.step = 0
# TODO: Is there a better way to get the maximum number of episodes?
self.max_episodes = np.ones(self.num_actors) * 1000
self.episode_lengths = np.zeros(self.num_actors, dtype=int)