def log_forward(self, input): # Get parameters and sizes W_e, W_x, W_h, W_y = self.parameters hidden_size = W_h.shape[0] nr_steps = input.shape[0] # Embedding layer z_e = W_e[input, :] # Recurrent layer h = np.zeros((nr_steps + 1, hidden_size)) for t in range(nr_steps): # Linear z_t = W_x.dot(z_e[t, :]) + W_h.dot(h[t, :]) # Non-linear h[t+1, :] = 1.0 / (1 + np.exp(-z_t)) # Output layer y = h[1:, :].dot(W_y.T) # Softmax log_p_y = y - logsumexp(y, axis=1, keepdims=True) return log_p_y, y, h, z_e, input
def log_forward(self, input): """Forward pass for sigmoid hidden layers and output softmax""" # Input tilde_z = input layer_inputs = [] # Hidden layers num_hidden_layers = len(self.parameters) - 1 for n in range(num_hidden_layers): # Store input to this layer (needed for backpropagation) layer_inputs.append(tilde_z) # Linear transformation weight, bias = self.parameters[n] z = np.dot(tilde_z, weight.T) + bias # Non-linear transformation (sigmoid) tilde_z = 1.0 / (1 + np.exp(-z)) # Store input to last layer layer_inputs.append(tilde_z) # Output linear transformation weight, bias = self.parameters[num_hidden_layers] z = np.dot(tilde_z, weight.T) + bias # Softmax is computed in log-domain to prevent underflow/overflow log_tilde_z = z - logsumexp(z, axis=1, keepdims=True) return log_tilde_z, layer_inputs
def log_forward(self, input=None): """Forward pass of the computation graph""" # Linear transformation z = np.dot(input, self.weight.T) + self.bias # Softmax implemented in log domain log_tilde_z = z - logsumexp(z, axis=1, keepdims=True) return log_tilde_z
def log_forward(self, input=None): """Forward pass of the computation graph""" # Linear transformation z = np.dot(input, self.weight.T) + self.bias # Softmax implemented in log domain log_tilde_z = z - logsumexp(z, axis=1, keepdims=True) return log_tilde_z