def forward(self, input):
mu = F.linear(input, self.weight, self.bias)
sigma = torch.sqrt(
F.linear(input**2,
torch.exp(self.wlog_sigma)**2) + 1e-8)
return mu + sigma * torch.randn_like(mu)
def LSTMCell(input, hidden, w_ih, w_hh, b_ih=None, b_hh=None):
"""
LSTM.weight_ih_l[k] = (4*hidden_size, input_size)
将三个gate的ih和输入的ih上下拼接合成一个矩阵weight_ih_l[k],
所以为什么是４倍hidden_size拼接顺序无所谓
同理 LSTM.weight_hh_l[k] = (4*hidden_size, hidden_size)，也是四个hh矩阵拼接合成一个矩阵
weight_ih =
[ih_in
ih_forget
ih_cell
ih_out
]
"""
hx, cx = hidden
gates = F.linear(input, w_ih, b_ih) + F.linear(hx, w_hh, b_hh)
# 此处沿着1轴切成４块，分别作为４个输入, 返回顺序无所谓，都是原始参数矩阵的输出
# 重要是之后的激活函数确定是那个门
# 分解出来的四个矩阵shape是(hidden_size, hidden_size)
ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1)
ingate = F.sigmoid(ingate)
forgetgate = F.sigmoid(forgetgate)
cellgate = F.tanh(cellgate)
outgate = F.sigmoid(outgate)
cy = (forgetgate * cx) + (ingate * cellgate)
hy = outgate * F.tanh(cy)
return hy, cy
def highwaynet(inputs, activation, units=128):
H = F.linear(inputs,
weight=torch.nn.init.normal_(
torch.empty(units, inputs.size(2))))
H = activation[0](H)
T = F.linear(inputs,
weight=torch.nn.init.normal_(
torch.empty(units, inputs.size(2))),
bias=nn.init.constant_(torch.empty(1, 1, units), -0.1))
T = activation[1](T)
return H * T + inputs * (1.0 - T)
def RNNTanhCell(input, hidden, w_ih, w_hh, b_ih=None, b_hh=None):
hy = F.tanh(F.linear(input, w_ih, b_ih) + F.linear(hidden, w_hh, b_hh))
return hy