def log_forward(self, input):
# Get parameters and sizes
W_e, W_x, W_h, W_y = self.parameters
hidden_size = W_h.shape[0]
nr_steps = input.shape[0]
# Embedding layer
z_e = W_e[input, :]
# Recurrent layer
h = np.zeros((nr_steps + 1, hidden_size))
for t in range(nr_steps):
# Linear
z_t = W_x.dot(z_e[t, :]) + W_h.dot(h[t, :])
# Non-linear
h[t+1, :] = 1.0 / (1 + np.exp(-z_t))
# Output layer
y = h[1:, :].dot(W_y.T)
# Softmax
log_p_y = y - logsumexp(y, axis=1, keepdims=True)
return log_p_y, y, h, z_e, input
def log_forward(self, input):
"""Forward pass for sigmoid hidden layers and output softmax"""
# Input
tilde_z = input
layer_inputs = []
# Hidden layers
num_hidden_layers = len(self.parameters) - 1
for n in range(num_hidden_layers):
# Store input to this layer (needed for backpropagation)
layer_inputs.append(tilde_z)
# Linear transformation
weight, bias = self.parameters[n]
z = np.dot(tilde_z, weight.T) + bias
# Non-linear transformation (sigmoid)
tilde_z = 1.0 / (1 + np.exp(-z))
# Store input to last layer
layer_inputs.append(tilde_z)
# Output linear transformation
weight, bias = self.parameters[num_hidden_layers]
z = np.dot(tilde_z, weight.T) + bias
# Softmax is computed in log-domain to prevent underflow/overflow
log_tilde_z = z - logsumexp(z, axis=1, keepdims=True)
return log_tilde_z, layer_inputs
def log_forward(self, input=None):
"""Forward pass of the computation graph"""
# Linear transformation
z = np.dot(input, self.weight.T) + self.bias
# Softmax implemented in log domain
log_tilde_z = z - logsumexp(z, axis=1, keepdims=True)
return log_tilde_z
def log_forward(self, input=None):
"""Forward pass of the computation graph"""
# Linear transformation
z = np.dot(input, self.weight.T) + self.bias
# Softmax implemented in log domain
log_tilde_z = z - logsumexp(z, axis=1, keepdims=True)
return log_tilde_z