def update_lr(self):
assert 'lr' in self.optim_param
old_lr = self.optim_param['lr']
self.optim_param['lr'] = old_lr * 0.9
logger.debug(
f'Learning rate decayed from {old_lr} to {self.optim_param["lr"]}')
self.optim = net_util.get_optim_multinet(self.params, self.optim_param)
def __init__(self,
in_dim,
hid_layers,
out_dim,
hid_layers_activation=None,
optim_param=None,
loss_param=None,
clamp_grad=False,
clamp_grad_val=1.0,
batch_norm=True):
'''
in_dim: dimension of the inputs
hid_layers: tuple consisting of two elements. (conv_hid, flat_hid)
Note: tuple must contain two elements, use empty list if no such layers.
1. conv_hid: list containing dimensions of the convolutional hidden layers. Asssumed to all come before the flat layers.
Note: a convolutional layer should specify the in_channel, out_channels, kernel_size, stride (of kernel steps), padding, and dilation (spacing between kernel points) E.g. [3, 16, (5, 5), 1, 0, (2, 2)]
For more details, see http://pytorch.org/docs/master/nn.html#conv2d and https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md
2. flat_hid: list of dense layers following the convolutional layers
out_dim: dimension of the ouputs
optim_param: parameters for initializing the optimizer
loss_param: measure of error between model
predictions and correct outputs
hid_layers_activation: activation function for the hidden layers
out_activation_param: activation function for the last layer
clamp_grad: whether to clamp the gradient
batch_norm: whether to add batch normalization after each convolutional layer, excluding the input layer.
@example:
net = ConvNet(
(3, 32, 32),
([[3, 36, (5, 5), 1, 0, (2, 2)],[36, 128, (5, 5), 1, 0, (2, 2)]],
[100]),
10,
hid_layers_activation='relu',
optim_param={'name': 'Adam'},
loss_param={'name': 'mse_loss'},
clamp_grad=False,
batch_norm=True)
'''
super(ConvNet, self).__init__()
# Create net and initialize params
self.in_dim = in_dim
self.out_dim = out_dim
self.batch_norm = batch_norm
self.conv_layers = []
self.conv_model = self.build_conv_layers(
hid_layers[0], hid_layers_activation)
self.flat_layers = []
self.dense_model = self.build_flat_layers(
hid_layers[1], out_dim, hid_layers_activation)
self.num_hid_layers = len(self.conv_layers) + len(self.flat_layers) - 1
self.init_params()
# Init other net variables
self.params = list(self.conv_model.parameters()) + \
list(self.dense_model.parameters())
self.optim = net_util.get_optim_multinet(self.params, optim_param)
self.loss_fn = net_util.get_loss_fn(self, loss_param)
self.clamp_grad = clamp_grad
self.clamp_grad_val = clamp_grad_val
def __init__(self,
in_dim,
hid_dim,
out_dim,
hid_layers_activation=None,
optim_param=None,
loss_param=None,
clamp_grad=False,
clamp_grad_val=1.0):
'''
in_dim: dimension of the inputs
hid_dim: list containing dimensions of the hidden layers
out_dim: list containing the dimensions of the ouputs
hid_layers_activation: activation function for the hidden layers
optim_param: parameters for initializing the optimizer
loss_param: measure of error between model predictions and correct outputs
clamp_grad: whether to clamp the gradient
@example:
net = MLPHeterogenousHeads(
1000,
[512, 256, 128],
[1, 1],
hid_layers_activation='relu',
optim_param={'name': 'Adam'},
loss_param={'name': 'mse_loss'},
clamp_grad=True,
clamp_grad_val=2.0)
'''
nn.Module.__init__(self)
# Create net and initialize params
self.in_dim = in_dim
self.out_dim = out_dim
self.layers = []
# Init network body
for i, layer in enumerate(hid_dim):
in_D = in_dim if i == 0 else hid_dim[i - 1]
out_D = hid_dim[i]
self.layers += [nn.Linear(in_D, out_D)]
self.layers += [net_util.get_activation_fn(hid_layers_activation)]
in_D = hid_dim[-1] if len(hid_dim) > 0 else in_dim
self.body = nn.Sequential(*self.layers)
# Init network output heads
self.out_layers = []
for i, dim in enumerate(out_dim):
self.out_layers += [nn.Linear(in_D, dim)]
self.layers += [self.out_layers]
self.init_params()
# Init other net variables
self.params = list(self.body.parameters())
for layer in self.out_layers:
self.params.extend(list(layer.parameters()))
self.optim_param = optim_param
self.optim = net_util.get_optim_multinet(self.params, self.optim_param)
self.loss_fn = net_util.get_loss_fn(self, loss_param)
self.clamp_grad = clamp_grad
self.clamp_grad_val = clamp_grad_val
def __init__(self,
in_dim,
hid_layers,
out_dim,
hid_layers_activation=None,
optim_param=None,
loss_param=None,
clamp_grad=False,
clamp_grad_val=1.0,
batch_norm=True):
'''
in_dim: dimension of the inputs
hid_layers: tuple consisting of two elements. (conv_hid, flat_hid)
Note: tuple must contain two elements, use empty list if no such layers.
1. conv_hid: list containing dimensions of the convolutional hidden layers. Asssumed to all come before the flat layers.
Note: a convolutional layer should specify the in_channel, out_channels, kernel_size, stride (of kernel steps), padding, and dilation (spacing between kernel points) E.g. [3, 16, (5, 5), 1, 0, (2, 2)]
For more details, see http://pytorch.org/docs/master/nn.html#conv2d and https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md
2. flat_hid: list of dense layers following the convolutional layers
out_dim: dimension of the output for one output, otherwise a list containing the dimensions of the ouputs for a multi-headed network
hid_layers_activation: activation function for the hidden layers
optim_param: parameters for initializing the optimizer
loss_param: measure of error between model predictions and correct outputs
clamp_grad: whether to clamp the gradient
batch_norm: whether to add batch normalization after each convolutional layer, excluding the input layer.
@example:
net = ConvNet(
(3, 32, 32),
([[3, 36, (5, 5), 1, 0, (2, 2)],[36, 128, (5, 5), 1, 0, (2, 2)]],[100]),
10,
hid_layers_activation='relu',
optim_param={'name': 'Adam'},
loss_param={'name': 'mse_loss'},
clamp_grad=False,
batch_norm=True)
'''
super(ConvNet, self).__init__()
# Create net and initialize params
# We need to transpose the dimensions for pytorch.
# OpenAI gym provides images as W x H x C, pyTorch expects C x W x H
self.in_dim = list(in_dim[:-1])
self.in_dim.insert(0, in_dim[-1])
# Handle multiple types of out_dim (single and multi-headed)
if type(out_dim) is int:
out_dim = [out_dim]
self.out_dim = out_dim
self.batch_norm = batch_norm
self.conv_layers = []
self.conv_model = self.build_conv_layers(hid_layers[0],
hid_layers_activation)
self.flat_layers = []
self.dense_model = self.build_flat_layers(hid_layers[1],
hid_layers_activation)
self.out_layers = []
in_D = hid_layers[1][-1] if len(hid_layers[1]) > 0 else self.flat_dim
for dim in out_dim:
self.out_layers += [nn.Linear(in_D, dim)]
self.num_hid_layers = len(self.conv_layers) + len(self.flat_layers)
self.init_params()
# Init other net variables
self.params = list(self.conv_model.parameters()) + \
list(self.dense_model.parameters())
for layer in self.out_layers:
self.params.extend(list(layer.parameters()))
self.optim_param = optim_param
self.optim = net_util.get_optim_multinet(self.params, self.optim_param)
self.loss_fn = net_util.get_loss_fn(self, loss_param)
self.clamp_grad = clamp_grad
self.clamp_grad_val = clamp_grad_val
def __init__(self,
in_dim,
hid_dim,
out_dim,
hid_layers_activation=None,
optim_param=None,
loss_param=None,
clamp_grad=False,
clamp_grad_val=1.0,
gpu=False):
'''
Multi state processing heads, single shared body, and multi action heads.
There is one state and action head per environment
Example:
Action env 1 Action env 2
_______|______ _______|______
| Act head 1 | | Act head 2 |
|______________| |______________|
| |
|__________________|
________________|_______________
| Shared body |
|________________________________|
|
________|_______
| |
_______|______ ______|_______
| State head 1 | | State head 2 |
|______________| |______________|
in_dim: list of lists containing dimensions of the state processing heads
hid_dim: list containing dimensions of the hidden layers
out_dim: list of lists containing dimensions of the ouputs
hid_layers_activation: activation function for the hidden layers
optim_param: parameters for initializing the optimizer
loss_param: measure of error between model predictions and correct outputs
clamp_grad: whether to clamp the gradient
gpu: whether to train using a GPU. Note this will only work if a GPU is available, othewise setting gpu=True does nothing
@example:
net = MLPNet(
[[800, 200],[400, 200]],
[100, 50, 25],
[[10], [15]],
hid_layers_activation='relu',
optim_param={'name': 'Adam'},
loss_param={'name': 'mse_loss'},
clamp_grad=True,
clamp_grad_val2.0,
gpu=False)
'''
super(MultiMLPNet, self).__init__()
# Create net and initialize params
self.in_dim = in_dim
self.out_dim = out_dim
self.state_heads_layers = []
self.state_heads_models = self.make_state_heads(
self.in_dim, hid_layers_activation)
self.shared_layers = []
self.body = self.make_shared_body(self.state_out_concat, hid_dim,
hid_layers_activation)
self.action_heads_layers = []
in_D = hid_dim[-1] if len(hid_dim) > 0 else self.state_out_concat
self.action_heads_models = self.make_action_heads(
in_D, self.out_dim, hid_layers_activation)
self.init_params()
if torch.cuda.is_available() and gpu:
for l in self.state_heads_models:
l.cuda()
self.body.cuda()
for l in self.action_heads_models:
l.cuda()
# Init other net variables
self.params = []
for model in self.state_heads_models:
self.params.extend(list(model.parameters()))
self.params.extend(list(self.body.parameters()))
for model in self.action_heads_models:
self.params.extend(list(model.parameters()))
self.optim_param = optim_param
self.optim = net_util.get_optim_multinet(self.params, self.optim_param)
self.loss_fn = net_util.get_loss_fn(self, loss_param)
self.clamp_grad = clamp_grad
self.clamp_grad_val = clamp_grad_val
def __init__(self,
in_dim,
hid_dim,
out_dim,
sequence_length,
hid_layers_activation=None,
optim_param=None,
loss_param=None,
clamp_grad=False,
clamp_grad_val=1.0,
num_rnn_layers=1):
'''
in_dim: dimension of the states
hid_dim: list containing dimensions of the hidden layers. The last element of the list is should be the dimension of the hidden state for the recurrent layer. The other elements in the list are the dimensions of the MLP (if desired) which is to transform the state space.
out_dim: dimension of the output for one output, otherwise a list containing the dimensions of the ouputs for a multi-headed network
sequence_length: length of the history of being passed to the net
hid_layers_activation: activation function for the hidden layers
optim_param: parameters for initializing the optimizer
loss_param: measure of error between model predictions and correct output
clamp_grad: whether to clamp the gradient
clamp_grad_val: what value to clamp the gradient at
num_rnn_layers: number of recurrent layers
@example:
net = RecurrentNet(
4,
[32, 64],
10,
8,
hid_layers_activation='relu',
optim_param={'name': 'Adam'},
loss_param={'name': 'mse_loss'},
clamp_grad=False)
'''
super(RecurrentNet, self).__init__()
# Create net and initialize params
self.in_dim = in_dim
self.sequence_length = sequence_length
self.hid_dim = hid_dim[-1]
# Handle multiple types of out_dim (single and multi-headed)
if type(out_dim) is int:
out_dim = [out_dim]
self.out_dim = out_dim
self.num_rnn_layers = num_rnn_layers
self.state_processing_layers = []
self.state_proc_model = self.build_state_proc_layers(
hid_dim[:-1], hid_layers_activation)
self.rnn_input_dim = hid_dim[-2] if len(hid_dim) > 1 else self.in_dim
self.rnn = nn.GRU(input_size=self.rnn_input_dim,
hidden_size=self.hid_dim,
num_layers=self.num_rnn_layers,
batch_first=True)
# Init network output heads
self.out_layers = []
for dim in self.out_dim:
self.out_layers += [nn.Linear(self.hid_dim, dim)]
self.layers = [self.state_processing_layers] + [self.rnn] + [self.out_layers]
self.num_hid_layers = None
self.init_params()
# Init other net variables
self.params = list(self.state_proc_model.parameters()) + list(self.rnn.parameters())
for layer in self.out_layers:
self.params.extend(list(layer.parameters()))
# Store named parameters for unit testing
self.named_params = list(self.state_proc_model.named_parameters()) + list(self.rnn.named_parameters())
for layer in self.out_layers:
self.named_params.extend(list(layer.named_parameters()))
self.optim_param = optim_param
self.optim = net_util.get_optim_multinet(self.params, self.optim_param)
self.loss_fn = net_util.get_loss_fn(self, loss_param)
self.clamp_grad = clamp_grad
self.clamp_grad_val = clamp_grad_val
