def __init__(self, num_inputs, num_hidden, forget_gate_bias=-1):
super(LayerNormLSTMCell, self).__init__()
self.forget_gate_bias = forget_gate_bias
self.num_hidden = num_hidden
self.fc_i2h = nn.Linear(num_inputs, 4 * num_hidden)
self.fc_h2h = nn.Linear(num_hidden, 4 * num_hidden)
self.ln_i2h = LayerNorm1D(4 * num_hidden)
self.ln_h2h = LayerNorm1D(4 * num_hidden)
self.ln_h2o = LayerNorm1D(num_hidden)
def __init__(self, model, num_layers, hidden_size):
super(MetaOptimizer, self).__init__()
self.meta_model = model
self.hidden_size = hidden_size
self.linear1 = nn.Linear(3, hidden_size)
self.ln1 = LayerNorm1D(hidden_size)
self.lstms = []
for i in range(num_layers):
self.lstms.append(LayerNormLSTMCell(hidden_size, hidden_size))
self.linear2 = nn.Linear(hidden_size, 1)
self.linear2.weight.data.mul_(0.1)
self.linear2.bias.data.fill_(0.0)