def test_softmax(self): result = af.softmax(np.array([[1, -1], [3, 4]])) expected_result = np.array([[0.119202, 0.006692], [0.880797, 0.993307]]) difference = result - expected_result self.assertTrue(np.linalg.norm(difference) < 1e-4) A = af.softmax(np.random.randn(5, 10)) self.assertTrue(np.linalg.norm(np.sum(A, axis=0, keepdims=True) - 1) < 1e-4)
def activation_forward(A_prev, W, b, activation_way): """ :param A_prev: :param W: :param b: :param activation_way: -- a text string indicate the way we activate this layer, "sigmoid","relu",... :return: A -- the activation output of this layer cache -- a dictionary contains "linear_cache' and "activation_cache" """ cache = dict() if activation_way == "relu": Z, linear_cache = linear_forward(A_prev, W, b) A, activation_cache = relu(Z) elif activation_way == "sigmoid": Z, linear_cache = linear_forward(A_prev, W, b) A, activation_cache = sigmoid(Z) elif activation_way == "tanh": Z, linear_cache = linear_forward(A_prev, W, b) A, activation_cache = tanh(Z) elif activation_way == "softmax": Z, linear_cache = linear_forward(A_prev, W, b) A, activation_cache = softmax(Z) cache["linear_cache"] = linear_cache cache["activation_cache"] = activation_cache return A, cache
def forward(self, x, t_label): """ x: input data """ self.t_label = t_label self.softmax_out = softmax(x) self.loss = _cross_entropy_calc(self.softmax_out, self.t_label) return self.loss
def mlp_fpass(data, w_1, b_1, w_2, b_2): """ Initializes the MLP weights using Xavier initialization (uniform) and the biases with zero """ z_1 = np.add(np.dot(data, w_1), b_1) a_1 = af.relu(z_1) z_2 = np.add(np.dot(a_1, w_2), b_2) a_2 = af.softmax(z_2) return z_1, a_1, z_2, a_2
def __run_network(self, registers: np.array, debug: dict = None) -> np.array: def take_params(values, i): """Return the next pair of weights and biases after the starting index and the new starting index.""" return values[i], values[i + 1], i + 2 # Extract the 0th (i.e. P( x = 0 )) component from all registers. last_hidden_layer = np.array(registers[:, 0][None, ...], dtype=np.float32) # Propogate forward to hidden layers. idx = 0 for i in range(self.context.num_hidden_layers): W, b, idx = take_params(self.context.network, idx) last_hidden_layer = relu(last_hidden_layer.dot(W) + b) controller_coefficients = [] for i, gate in enumerate(self.context.gates): coeffs = [] for j in range(gate.arity): W, b, idx = take_params(self.context.network, idx) coeff = softmax(last_hidden_layer.dot(W) + b) coeffs.append(coeff) controller_coefficients.append(coeffs) # Forward propogate to new register value coefficients. for i in range(self.context.num_regs): W, b, idx = take_params(self.context.network, idx) coeff = softmax(last_hidden_layer.dot(W) + b) controller_coefficients.append(coeff) # Forward propogate to generate willingness to complete. W, b, idx = take_params(self.context.network, idx) complete = sigmoid(last_hidden_layer.dot(W) + b) if debug is not None: debug.fi = np.around(complete.sum(), 3) return controller_coefficients, complete
def output_layer(x, w, b): """output layer 多層パーセプトロンの出力層 Args: x: input w: wight b: bias Retuens: output of mlp """ u = np.dot(x, w) + b return softmax(u)
def feedforward(self, input): input = np.array(input) self.nodesave = list(range(len(self.hidden_w) + 2)) # input self.nodesave[0] = self.actifunc( self.input_w.dot(input) + self.input_b) # hidden for i, w, b in zip(range(len(self.hidden_w)), self.hidden_w, self.hidden_b): self.nodesave[i + 1] = self.actifunc(w.dot(self.nodesave[i]) + b) # output self.nodesave[-1] = activation_functions.softmax( self.output_w.dot(self.nodesave[-2]) + self.output_b) return self.nodesave[-1]
def feedforward(self, dropout_rate=[]): for layer in range(1, self.number_of_layers): #iterates through every layer, skipping the input layer. if (layer == self.number_of_layers - 1 and self.act_funcs[-1] == "softmax"): #softmax has no weights or biases. self.z_values[-1] = self.activations[-2] - self.activations[-2].max() #computational trick to avoid overflow. self.activations[-1] = af.softmax(self.z_values[-1]) else: activate = af.func_dict(0)[self.act_funcs[layer-1]] if (self.validating == True): if (not dropout_rate) or (dropout_rate[layer-1] == 0): scale = 1 else: scale = (1 - dropout_rate[layer-1]) #present with probability 1-p. self.z_values[layer] = np.dot(self.weights[layer-1]*scale, self.activations[layer-1]) + self.biases[layer] self.activations[layer] = activate(self.z_values[layer]) else: self.z_values[layer] = np.matmul(self.weights[layer-1], self.activations[layer-1]) + self.biases[layer] self.activations[layer] = activate(self.z_values[layer]) if (dropout_rate) and (dropout_rate[layer-1] != 0): self.activations[layer], self.z_values[layer] = reg.dropout(self.activations[layer], self.z_values[layer], dropout_rate[layer-1], self.parameters[layer])
def forward(self, incoming_x): # incoming data should be (sentence length, projected_dimension) print('x shape into head is', incoming_x.shape) n_sents, vector_dimension, projected_dimension = incoming_x.shape assert vector_dimension == self.ni Q = self.query_layer.forward(incoming_x) K = self.key_layer.forward(incoming_x) V = self.value_layer.forward(incoming_x) # scaled dot product score = (Q @ K.T) / np.sqrt(self.no) #MASK would occur here, #self.mask[:, :, :T, :T] == 0, -np.inf) score = activation.softmax(score) self.score = self.dropout.forward(score, self.training_now) self.output = self.score @ V return self.output
def softmax_ce_derivation(y_hat, y): #derivation of CE(softmax(y),y_hat) respect to y return (activation_functions.softmax(y_hat) - y).T
b_hidden_2 = np.zeros(num_hidden_units_2) w_out_hidden = np.random.normal(0, 1 / (num_features**0.5), (num_hidden_units_2, num_output)) b_out_hidden = np.zeros(num_output) # Training --------------------------------------------------------------------------------------------------------------------------- for e in range(epochs + 1): for b in range(data_size // batch_size): X, Y = shuffle_and_get_batch_data(X_data, Y_data, batch_size) # Forward propagation hidden_output_1 = activation(np.dot(X, w_hidden_1) + b_hidden_1) hidden_output_2 = activation( np.dot(hidden_output_1, w_hidden_2) + b_hidden_2) nn_output = softmax( np.dot(hidden_output_2, w_out_hidden) + b_out_hidden) # Backward propagation # Step back through the network and calculate errors and deltas for the weights and biases hidden_out_error = Y - nn_output d_w_out_hidden = np.dot(hidden_output_2.T, hidden_out_error) hidden_2_error = np.multiply( np.matmul(hidden_out_error, w_out_hidden.T), activation_d(hidden_output_2)) d_w_hidden_2 = np.dot(hidden_output_1.T, hidden_2_error) hidden_in_error = np.multiply(np.matmul(hidden_2_error, w_hidden_2.T), activation_d(hidden_output_1)) d_w_in_hidden = np.dot(X.T, hidden_in_error)
def probabilities_from_activation(self, vmap): return activation_functions.softmax(vmap[self])
def probabilities_from_activation(self, vmap): return activation_functions.softmax(vmap[self])
def loss(self, X, y=None, reg=0.0): W1, b1 = self.params['W1'], self.params['b1'] W2, b2 = self.params['W2'], self.params['b2'] N, D = X.shape """ Get scores and perform forward pass """ scores = None # Layer 1 # (N, D)x(D, H) + (1, H) = (N, H) scores = np.dot(X, W1) + b1 relu_activations = ReLU(scores) # Dropout after layer 1 # Kill whole neurons # drop1 = np.random.randn(scores.shape) # relu_activations[:, drop1 <= self.dropout_p] = 0 # Layer 2 scores = relu_activations.dot(W2) + b2 if y is None: return scores softmax_scores = softmax(scores) scores = -np.log(softmax_scores[range(N), y]) loss = np.mean(scores) loss += reg * (np.sum(W1 * W1) + np.sum(W2 * W2) + np.sum(b1 * b1) + np.sum(b2 * b2)) """ Calculating gradients via backprop """ grads = {} # Differentiate loss wrt scores for each class dsoft = softmax_scores.copy() dsoft[range(N), y] -= 1 dsoft /= N dW2 = relu_activations.T.dot(dsoft) dW2 += 2 * reg * W2 grads['dW2'] = dW2 db2 = dsoft * 1 grads['db2'] = np.sum(db2, axis=0) dx2 = np.dot(dsoft, W2.T) relu_ones = (relu_activations >= 0) * 1 # Only allow those gradients to flow back whose activations were positive drelu = dx2 * relu_ones dW1 = X.T.dot(drelu) dW1 += 2 * reg * W1 grads['dW1'] = dW1 db1 = drelu * 1 grads['db1'] = np.sum(db1, axis=0) dimage = drelu.dot(W1.T) grads['dimage'] = dimage return loss, grads