def __init__(self, obs_space, action_space, num_outputs, model_config, name, **custom_model_config):
#call inheritance
TorchModelV2.__init__(self, obs_space, action_space, num_outputs, model_config, name)
nn.Module.__init__(self)
HP = custom_model_config["hyperparameters"]
#check if HP is of type LSTMHyperParameters
if not isinstance(HP, self.hyperparameter_type):
raise Exception("Make sure to pass the correct HyperParameter-Type for your model")
#save values
self.feature_size = len(HP.features)
self.hidden_size = HP.hidden_size
self.num_layers = HP.num_layers
self.dropout = HP.dropout
#create the lstm layer
self.lstm1 = nn.LSTM(input_size=self.feature_size, hidden_size=self.hidden_size, batch_first=True, num_layers=self.num_layers, dropout=self.dropout)
#create the activation
if HP.activation is hp.Activation.TANH:
self.activation = nn.Tanh()
elif HP.activation is hp.Activation.RELU:
self.activation = nn.ReLU()
#create the linear layers
self.linear = nn.Linear(self.hidden_size, 3)
#create the valuefuntion
self.vf = nn.Linear(self.hidden_size, 1)
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
h, w, c = obs_space.shape
shape = (c, h, w)
conv_seqs = []
for out_channels in [16, 32, 32]:
conv_seq = ConvSequence(shape, out_channels)
shape = conv_seq.get_output_shape()
conv_seqs.append(conv_seq)
self.conv_seqs = nn.ModuleList(conv_seqs)
self._embed_shape = (shape[0], shape[1], shape[2])
self._n_embed_shape = (self._embed_shape[0] * num_outputs, ) + self._embed_shape[1:]
self.hidden_fc = nn.Linear(in_features=shape[0] * shape[1] * shape[2], out_features=256)
self.logits_fc = nn.Linear(in_features=256, out_features=num_outputs)
self.value_fc = nn.Linear(in_features=256, out_features=1)
self.rew_hid = nn.Linear(in_features=shape[0] * shape[1] * shape[2], out_features=256)
self.rew_out = nn.Linear(in_features=256, out_features=num_outputs)
self.trans_conv = nn.Conv2d(in_channels=shape[0]*num_outputs, out_channels=shape[0]*num_outputs,
kernel_size=3, padding=1, stride=1, groups=num_outputs)
def __init__(self, obs_space, action_space, num_outputs, model_config,
name, num_objects, object_state_size):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
h, w, c = obs_space.shape
shape = (c, h, w)
# Is this channel first or last format?
# Seems like the original model assumes NCHW, why?
# i.e., it flips it below; weird
self.obs_encoder = ObsEncoder(input_dim=c,
hidden_dim=16,
num_objects=num_objects)
self.mask_encoder = MaskEncoder(input_dim=num_objects,
hidden_dim=16,
fc_hidden_dim=128,
output_dim=object_state_size)
self.gnn = GNN(input_dim=object_state_size,
hidden_dim=16,
num_objects=num_objects)
gnn_flat_output_dim = num_objects * object_state_size
output_hidden_dim = 128
self.hidden_fc = nn.Linear(in_features=gnn_flat_output_dim,
out_features=output_hidden_dim)
self.logits_fc = nn.Linear(in_features=output_hidden_dim,
out_features=num_outputs)
self.value_fc = nn.Linear(in_features=output_hidden_dim,
out_features=1)
self._obj_masks = None
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
self.obs_size = _get_size(obs_space)
self.rnn_hidden_dim = model_config["lstm_cell_size"]
self.free_log_std = model_config.get("free_log_std")
if self.free_log_std:
assert num_outputs % 2 == 0, (
"num_outputs must be divisible by two", num_outputs)
num_outputs = num_outputs // 2
if self.free_log_std:
self.log_std = torch.nn.Parameter(torch.as_tensor([0.0] * num_outputs))
self.fc1 = nn.Linear(self.obs_size, self.rnn_hidden_dim)
self.rnn = nn.GRU(self.rnn_hidden_dim, self.rnn_hidden_dim, batch_first=True)
self.fc2 = nn.Linear(self.rnn_hidden_dim, num_outputs)
self.mu = nn.Linear(self.rnn_hidden_dim, 7)
self.alpha = nn.Sequential(
nn.Linear(self.rnn_hidden_dim, 7),
nn.Softmax(dim=-1)
)
# self.value_branch = nn.Linear(self.rnn_hidden_dim, 1)
self._cur_value = None
def __init__(
self,
obs_space,
action_space,
num_outputs,
model_config,
name,
**kwargs,
):
orig_space = getattr(obs_space, "original_space", obs_space)
assert (isinstance(orig_space, Dict)
and "action_mask" in orig_space.spaces
and "observations" in orig_space.spaces)
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name, **kwargs)
nn.Module.__init__(self)
self.internal_model = TorchFC(
orig_space["observations"],
action_space,
num_outputs,
model_config,
name + "_internal",
)
# disable action masking --> will likely lead to invalid actions
self.no_masking = False
if "no_masking" in model_config["custom_model_config"]:
self.no_masking = model_config["custom_model_config"]["no_masking"]
def __init__(self, observation_space, action_space, num_outputs,
model_config, name):
TorchModelV2.__init__(self, observation_space, action_space,
num_outputs, model_config, name)
nn.Module.__init__(self)
# Non-shared initial layer.
self.first_layer = SlimFC(
int(np.product(observation_space.shape)),
64,
activation_fn=nn.ReLU,
initializer=torch.nn.init.xavier_uniform_)
# Non-shared final layer.
self.last_layer = SlimFC(
64,
self.num_outputs,
activation_fn=None,
initializer=torch.nn.init.xavier_uniform_)
self.vf = SlimFC(
64,
1,
activation_fn=None,
initializer=torch.nn.init.xavier_uniform_,
)
self._global_shared_layer = TORCH_GLOBAL_SHARED_LAYER
self._output = None
def __init__(self, obs_space, action_space, num_outputs, model_config, name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs, model_config, name)
nn.Module.__init__(self)
self.counter = 0
self.nn_layers = nn.ModuleList()
self.conv_filters_dict = model_config["custom_model_config"]["conv_filters"]
input_c = model_config["custom_model_config"]["inChannel"]
for conv in range(len(self.conv_filters_dict)):
self.nn_layers.append(nn.Conv2d(input_c, self.conv_filters_dict[conv][0], kernel_size= self.conv_filters_dict[conv][1], stride=self.conv_filters_dict[conv][2], padding=self.conv_filters_dict[conv][3] ))
if self.conv_filters_dict[conv][4][0] == 1:
self.nn_layers.append(nn.MaxPool2d(kernel_size=self.conv_filters_dict[conv][4][1], stride=self.conv_filters_dict[conv][4][2]))
self.counter += 1
input_c = self.conv_filters_dict[conv][0]
self.counter += 1
if(model_config["custom_model_config"]["conv_activation"] == "relu"):
self.conv_activ = F.relu
self.fc_hidden_dict = model_config["custom_model_config"]["fcnet_hiddens"]
input_c = 1
for i in range(len(self.fc_hidden_dict)) :
self.nn_layers.append(nn.Linear(self.fc_hidden_dict[i][0],self.fc_hidden_dict[i][1]))
if(model_config["custom_model_config"]["fcnet_activation"] == "relu"):
self.fully_connect_activation = F.relu
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
"""
The model of the network will be defined here
"""
self.obs_space = obs_space
self.action_space = action_space
self.model_config = model_config
self.name = name
self.network_size = model_config["custom_model_config"]["network_size"]
if isinstance(self.obs_space, Box):
self.obs_shape = obs_space.shape[0]
else:
self.obs_shape = self.obs_space
self.layers = nn.Sequential()
last_size = self.obs_space.shape[0]
i = 0
for layer_size in self.network_size:
self.layers.add_module("linear_{}".format(i),
nn.Linear(last_size, layer_size))
self.layers.add_module("relu_{}".format(i), nn.ReLU())
last_size = layer_size
i += 1
self.layers.add_module("linear_{}".format(i),
nn.Linear(last_size, num_outputs))
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
filters = model_config.get("conv_filters")
if not filters:
filters = _get_filter_config(obs_space.shape)
layers = []
(w, h, in_channels) = obs_space.shape
in_size = [w, h]
for out_channels, kernel, stride in filters[:-1]:
padding, out_size = valid_padding(in_size, kernel,
[stride, stride])
layers.append(
SlimConv2d(in_channels, out_channels, kernel, stride, padding))
in_channels = out_channels
in_size = out_size
out_channels, kernel, stride = filters[-1]
layers.append(
SlimConv2d(in_channels, out_channels, kernel, stride, None))
self._convs = nn.Sequential(*layers)
self._logits = SlimFC(out_channels,
num_outputs,
initializer=nn.init.xavier_uniform_)
self._value_branch = SlimFC(out_channels,
1,
initializer=normc_initializer())
self._cur_value = None
def __init__(self, obs_space, action_space, num_outputs, model_config, name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs, model_config, name)
nn.Module.__init__(self)
self.action_model = action_model
self.value_model = value_model
self._model_in = None
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
self.obs_size = _get_size(obs_space)
self.hidden_size = model_config["hidden_size"]
# number of discrete actions
self.num_category = model_config["num_category"]
# number of parameters for each action
self.num_params = model_config['num_params']
self.rnn = nn.GRUCell(self.obs_size, self.hidden_size)
# category head (for discrete action)
self.h2c = nn.Sequential(
nn.Linear(self.hidden_size, self.hidden_size), nn.Tanh(),
nn.Linear(self.hidden_size, self.num_category))
# parameter head (for continuous parameters of the action)
self.h2p = nn.Sequential(nn.Linear(self.hidden_size, self.hidden_size),
nn.Tanh(),
nn.Linear(self.hidden_size, self.num_params))
# critic
self.critic = nn.Sequential(
nn.Linear(self.hidden_size, self.hidden_size), nn.Tanh(),
nn.Linear(self.hidden_size, 1))
def __init__(
self,
obs_space: gym.spaces.Space,
action_space: gym.spaces.Space,
num_outputs: int,
model_config: ModelConfigDict,
name: str,
):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
self._is_action_discrete = isinstance(action_space,
gym.spaces.Discrete)
# TODO: I don't know why this is true yet? (in = num_outputs)
self.obs_ins = num_outputs
self.action_dim = np.product(self.action_space.shape)
self.actor_model = self._build_actor_net("actor")
twin_q = self.model_config["twin_q"]
self.q_model = self._build_q_net("q")
if twin_q:
self.twin_q_model = self._build_q_net("twin_q")
else:
self.twin_q_model = None
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
custom_model_config = model_config.get("custom_model_config")
# activation = custom_model_config.get("activation")
# no_final_layer = custom_model_config.get("no_final_layer")
sse_hiddens = [256, 128, 64]
hiddens = [256, 256, 256, 32]
self.num_slice = 3
self.vf_share_layers = True
sse_layers = []
prev_size = int(np.product(obs_space.shape) / self.num_slice)
for i, size in enumerate(sse_hiddens):
if i is not len(sse_hiddens) - 1:
sse_layers.append(nn.Linear(prev_size, size))
sse_layers.append(nn.Tanh())
else:
sse_layers.append(nn.Linear(prev_size, size))
prev_size = size
self.sse_encode_block = nn.Sequential(*sse_layers)
prev_size *= self.num_slice
hidden_layers = []
for i, size in enumerate(hiddens):
hidden_layers.append(nn.Linear(prev_size, size))
hidden_layers.append(nn.LeakyReLU(0.1))
prev_size = size
self.hidden_layers = nn.Sequential(*hidden_layers)
self.logits = nn.Linear(prev_size, num_outputs)
if self.vf_share_layers:
self.value_branch = nn.Linear(prev_size, 1)
else:
raise NotImplementedError
self.features = None
def __init__(self, obs_space: gym.spaces.Space,
action_space: gym.spaces.Space, num_outputs: int,
model_config: ModelConfigDict, name: str):
if not model_config.get("conv_filters"):
raise ValueError("Config for conv_filters is required")
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
activation = self.model_config.get("conv_activation")
filters = self.model_config["conv_filters"]
assert len(filters) > 0,\
"Must provide at least 1 entry in `conv_filters`!"
# Whether the last layer is the output of a Flattened (rather than
# a n x (1,1) Conv2D).
self.last_layer_is_flattened = False
self._logits = None
# Holds the current "base" output (before logits layer).
self._features = None
self.num_outputs = num_outputs if num_outputs else action_space.shape[0]
self.filters = filters
self.activation = activation
self.obs_space = obs_space
self._create_model()
def __init__(
self,
obs_space,
action_space,
num_outputs,
model_config,
name,
# customs
embed_dim=256,
encoder_type="impala",
**kwargs):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
self.action_dim = action_space.n
self.discrete = True
self.action_outs = q_outs = self.action_dim
self.action_ins = None # No action inputs for the discrete case.
self.embed_dim = embed_dim
h, w, c = obs_space.shape
shape = (c, h, w)
# obs embedding
self.encoder = make_encoder(encoder_type,
shape,
out_features=embed_dim)
self.logits_fc = nn.Linear(in_features=embed_dim,
out_features=num_outputs)
self.value_fc = nn.Linear(in_features=embed_dim, out_features=1)
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, custom_input_space, action_space,
num_outputs, model_config, name)
nn.Module.__init__(self)
self.torch_sub_model = TorchFC(custom_input_space, action_space,
num_outputs, model_config, name)
prev_safe_layer_size = int(np.product(custom_input_space.shape))
vf_layers = []
activation = model_config.get("fcnet_activation")
hiddens = [32]
for size in hiddens:
vf_layers.append(
SlimFC(in_size=prev_safe_layer_size,
out_size=size,
activation_fn=activation,
initializer=normc_initializer(1.0)))
prev_safe_layer_size = size
vf_layers.append(
SlimFC(in_size=prev_safe_layer_size,
out_size=1,
initializer=normc_initializer(0.01),
activation_fn=None))
self.safe_branch_separate = nn.Sequential(*vf_layers)
self.last_in = None
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
custom_config = model_config["custom_options"]
latent_size = custom_config['latent_size']
self.main = Encoder(latent_size=latent_size)
if custom_config['encoder_path'] is not None:
# saved checkpoints could contain extra weights such as linear_logsigma
weights = torch.load(custom_config['encoder_path'],
map_location=torch.device('cpu'))
for k in list(weights.keys()):
if k not in self.main.state_dict().keys():
del weights[k]
self.main.load_state_dict(weights)
print("Loaded Weights")
else:
print("No Load Weights")
self.critic = nn.Sequential(nn.Linear(latent_size, 400), nn.ReLU(),
nn.Linear(400, 300), nn.ReLU(),
nn.Linear(300, 1))
self.actor = nn.Sequential(nn.Linear(latent_size, 400), nn.ReLU(),
nn.Linear(400, 300), nn.ReLU())
self.alpha_head = nn.Sequential(nn.Linear(300, 3), nn.Softplus())
self.beta_head = nn.Sequential(nn.Linear(300, 3), nn.Softplus())
self._cur_value = None
self.train_encoder = custom_config['train_encoder']
print("Train Encoder: ", self.train_encoder)
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
# c, h, w = obs_space.shape # Frame stack
h, w, c = obs_space.shape
shape = (c, h, w)
embed_size = 256
conv_seqs = []
for out_channels in [16, 32, 32]:
conv_seq = ConvSequence(shape, out_channels)
shape = conv_seq.get_output_shape()
conv_seqs.append(conv_seq)
self.conv_seqs = nn.ModuleList(conv_seqs)
self.hidden_fc_1 = nn.Linear(in_features=shape[0] * shape[1] * shape[2], out_features=embed_size)
# self.hidden_fc_2 = nn.Linear(in_features=256, out_features=256)
self.logits_fc = nn.Linear(in_features=256, out_features=num_outputs)
self.value_fc = nn.Linear(in_features=256, out_features=1)
# ICM Layers
self.idm_hidden = nn.Linear(in_features=embed_size * 2, out_features=256)
self.idm_logits = nn.Linear(in_features=256, out_features=num_outputs)
self.fdm_hidden = nn.Linear(in_features=embed_size + num_outputs, out_features=256)
self.fdm_output = nn.Linear(in_features=256, out_features=embed_size)
def __init__(self, obs_space: gym.spaces.Space, action_space: gym.spaces.Space, num_outputs: int,
model_config: ModelConfigDict, name: str, base_model: ModelV2):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs, model_config, name)
nn.Module.__init__(self)
self.activation = base_model.model_config.get("conv_activation")
self.output_size = base_model.num_outputs
self.base_model = base_model
filters = self.model_config["conv_filters"]
out_channels, kernel, stride = filters[-1]
(w, h, in_channels) = obs_space.shape
in_size = [w, h]
self.prediction = MuZeroPredictionModel(self.activation, in_size, kernel, stride, self.output_size)
self.dynamics = MuZeroDynamicsModel(self.activation, self.output_size)
out_conv = SlimConv2d(
out_channels,
out_channels,
kernel=1,
stride=1,
padding=None,
activation_fn=None
)
self.representation = nn.Sequential(base_model._convs, out_conv) # assumes you're using vision network not fc
self.hidden = None
def __init__(self, obs_space: gym.spaces.Space,
action_space: gym.spaces.Space, num_outputs: int,
model_config: ModelConfigDict, name: str):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
custom_configs = model_config.get("custom_model_config")
self._sensor_seq_len = custom_configs.get("sensor_seq_len", 10)
activation = model_config.get("fcnet_activation", "tanh")
encoder_layer = nn.TransformerEncoderLayer(d_model=3, nhead=3, batch_first=True, dim_feedforward=128)
self._transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=2)
self._all_fc1 = SlimFC(in_size=3,
out_size=64,
initializer=normc_initializer(1.0),
activation_fn=activation)
self._all_fc2 = SlimFC(in_size=64,
out_size=16,
initializer=normc_initializer(1.0),
activation_fn=activation)
self._action_layer = SlimFC(in_size=16,
out_size=num_outputs,
initializer=normc_initializer(0.01),
activation_fn=None)
self._value_layer = SlimFC(in_size=16,
out_size=1,
initializer=normc_initializer(0.01),
activation_fn=None)
self._features = None
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
h, w, c = obs_space.shape
shape = (c, h, w)
conv_seqs = []
for out_channels in [16, 32, 32]:
conv_seq = ConvSequence(shape, out_channels)
shape = conv_seq.get_output_shape()
conv_seqs.append(conv_seq)
self.conv_seqs = nn.ModuleList(conv_seqs)
self.hidden_fc = nn.Linear(in_features=shape[0] * shape[1] * shape[2],
out_features=256)
self.hidden_fc.weight.data *= 1.4 / self.hidden_fc.weight.norm(
dim=1, p=2, keepdim=True)
self.hidden_fc.bias.data *= 0
self.logits_fc = nn.Linear(in_features=256, out_features=num_outputs)
self.logits_fc.weight.data *= 0.1 / self.logits_fc.weight.norm(
dim=1, p=2, keepdim=True)
self.logits_fc.bias.data * 0
self.value_fc = nn.Linear(in_features=256, out_features=1)
self.value_fc.weight.data *= 0.1 / self.value_fc.weight.norm(
dim=1, p=2, keepdim=True)
self.value_fc.bias.data *= 0
self.aux_vf_head = NormedLinear(256, 1, scale=0.1)
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
num_options = model_config.get('oc_num_options')
TorchModelV2.__init__(self, obs_space, action_space,
num_outputs * num_options, model_config, name)
nn.Module.__init__(self)
activation = get_activation_fn(model_config.get("fcnet_activation"),
framework="torch")
hiddens = model_config.get("fcnet_hiddens")
self.option_epsilon = model_config.get('oc_option_epsilon')
layers = []
prev_layer_size = int(np.product(obs_space.shape))
# Create layers
for size in hiddens:
layers.append(nn.Linear(prev_layer_size, size))
layers.append(activation)
prev_layer_size = size
self._body = nn.Sequential(*layers)
self.q = nn.Linear(prev_layer_size,
num_options) # Value for each option
self.pi = nn.Sequential(
nn.Linear(prev_layer_size, num_options * num_outputs),
View((num_options, num_outputs)),
nn.Softmax(dim=-1)) # Action probabilities for each option
self.beta = nn.Sequential(nn.Linear(prev_layer_size, num_options),
nn.Sigmoid) # Termination probabilities
# Holds the current "base" output (before logits layer).
self._features = self._q = self._v = self._pi = self._beta = None
# Holds the last input, in case value branch is separate.
self._last_flat_in = None
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
self.obs_space = obs_space
self.action_space = action_space
self.model_config = model_config
self.name = name
if isinstance(self.obs_space, Box):
self.obs_shape = obs_space.shape[0]
else:
self.obs_shape = self.obs_space
# DQN Network:
self.layers = nn.Sequential()
self.layers.add_module("linear_1", nn.Linear(self.obs_space.shape[0], 32))
self.layers.add_module("relu_1", nn.ReLU())
self.layers.add_module("linear_2", nn.Linear(32, 64))
self.layers.add_module("relu_2", nn.ReLU())
self.layers.add_module("linear_3", nn.Linear(64, 32))
self.layers.add_module("relu_3", nn.ReLU())
self.layers.add_module("linear_4", nn.Linear(32, num_outputs))
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
custom_config = model_config['custom_options']
self.main = Encoder()
if custom_config['encoder_path'] is not None:
print("Load Trained Encoder")
# saved checkpoints could contain extra weights such as linear_logsigma
weights = torch.load(custom_config['encoder_path'],
map_location={'cuda:0': 'cpu'})
for k in list(weights.keys()):
if k not in self.main.state_dict().keys():
del weights[k]
self.main.load_state_dict(weights)
self.critic = nn.Sequential(nn.Linear(32, 1024), nn.ReLU(),
nn.Linear(1024, 256), nn.ReLU(),
nn.Linear(256, 1))
self.actor = nn.Sequential(nn.Linear(32, 1024), nn.ReLU(),
nn.Linear(1024, 256), nn.ReLU(),
nn.Linear(256, 3), nn.Sigmoid())
self.actor_logstd = nn.Parameter(torch.zeros(3), requires_grad=True)
self._cur_value = None
print("Train Encoder:", custom_config['train_encoder'])
self.train_encoder = custom_config['train_encoder']
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
num_options = model_config.get('oc_num_options')
TorchModelV2.__init__(self, obs_space, action_space, num_outputs * num_options,
model_config, name)
nn.Module.__init__(self)
layers = []
(w, h, in_channels) = obs_space.shape
in_size = [w, h]
# Convolutional layers
for out_channels, kernel, stride in OCNET_FILTERS:
padding, out_size = same_padding(in_size, kernel, [stride, stride])
layers.append(nn.Conv2d(in_channels, out_channels, kernel, stride, padding))
layers.append(nn.ReLU())
in_channels = out_channels
in_size = out_size
# Dense layer after flattening output, using ReLU
hSize = OCNET_DENSE
self.option_epsilon = model_config.get('oc_option_epsilon')
layers.append(nn.Flatten())
layers.append(nn.Linear(in_size, hSize))
layers.append(nn.ReLU())
self._convs = nn.Sequential(*layers)
# q, pi, beta, and v
self.q = nn.Linear(hSize, num_options) # Value for each option
#self.v = nn.Linear(hSize, 1) # Value for state alone? Or do
self.pi = nn.Sequential(nn.Linear(hSize, num_options * num_outputs), View((num_options, num_outputs)), nn.Softmax(dim=-1)) # Action probabilities for each option
self.beta = nn.Sequential(nn.Linear(hSize, num_options), nn.Sigmoid) # Termination probabilities
# Holds the current "base" output (before heads).
self._features = self._q = self._v = self._pi = self._beta = None
def __init__(self, obs_space, action_space, num_outputs, model_config,
name, **kwargs):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
layers = []
prev_layer_size = int(np.product(obs_space.shape))
self._logits = None
# Create layers 0 to second-last.
for size in [256, 256]:
layers.append(
SlimFC(in_size=prev_layer_size,
out_size=size,
initializer=torch_normc_initializer(1.0),
activation_fn=nn.ReLU))
prev_layer_size = size
# Add a batch norm layer.
layers.append(nn.BatchNorm1d(prev_layer_size))
self._logits = SlimFC(in_size=prev_layer_size,
out_size=self.num_outputs,
initializer=torch_normc_initializer(0.01),
activation_fn=None)
self._value_branch = SlimFC(in_size=prev_layer_size,
out_size=1,
initializer=torch_normc_initializer(1.0),
activation_fn=None)
self._hidden_layers = nn.Sequential(*layers)
self._hidden_out = None
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
print(model_config)
hiddens = model_config.get("fcnet_hiddens")
activation = _get_activation_fn(model_config.get("fcnet_activation"))
logger.debug("Constructing fcnet {} {}".format(hiddens, activation))
layers = []
last_layer_size = np.product(obs_space.shape)
for size in hiddens:
layers.append(
SlimFC(in_size=last_layer_size,
out_size=size,
initializer=normc_initializer(1.0),
activation_fn=activation))
last_layer_size = size
self._hidden_layers = nn.Sequential(*layers)
self._logits = SlimFC(in_size=last_layer_size,
out_size=num_outputs,
initializer=normc_initializer(0.01),
activation_fn=None)
self._value_branch = SlimFC(in_size=last_layer_size,
out_size=1,
initializer=normc_initializer(1.0),
activation_fn=None)
self._cur_value = None
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
self.obs_space = obs_space
# Conv-net blovk
self.conv1 = torch.nn.Conv2d(16,
32,
kernel_size=3,
stride=1,
padding=1)
self.pool = torch.nn.MaxPool2d(kernel_size=2, stride=2, padding=0)
self.conv2 = torch.nn.Conv2d(32,
32,
kernel_size=3,
stride=1,
padding=1)
# Calculating output neurons after conv layers
self.neurons = self.linear_input_neurons()
# FC-net block
self.fc1 = torch.nn.Linear(self.neurons, 1024)
self.fc2 = torch.nn.Linear(1024, 256)
# Value function branch
self.value_function_fc = nn.Linear(256, 1)
# Advantage function branch
self.advantage_function_fc = nn.Linear(256, num_outputs)
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
self.torch_sub_model = TorchFC(obs_space, action_space, num_outputs,
model_config, name)
def __init__(self, obs_space, action_space, num_outputs, model_config,
name):
TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
model_config, name)
nn.Module.__init__(self)
self.fc = FullyConnectedNetwork(
obs_space.original_space.child_space["location"], action_space,
num_outputs, model_config, name)