def train(x, y, iterations=10000, learning_rate=0.1): global W1, W2, W3, B1, B2, B3, error m = x.shape[0] error = [] for _ in range(iterations): a0, z1, a1, z2, a2, z3, a3 = forward(x, predict=False) da3 = a3 - y.T dz3 = da3 * sigmoid(z3, derivative=True) dw3 = dz3.dot(a2.T) / m db3 = np.sum(dz3, axis=1, keepdims=True) / m da2 = W3.T.dot(dz3) dz2 = np.multiply(da2, sigmoid(z2, derivative=True)) dw2 = dz2.dot(a1.T) / m db2 = np.sum(dz2, axis=1, keepdims=True) / m da1 = W2.T.dot(dz2) dz1 = np.multiply(da1, sigmoid(z1, derivative=True)) dw1 = dz1.dot(a0.T) / m db1 = np.sum(dz1, axis=1, keepdims=True) / m W1 -= learning_rate * dw1 B1 -= learning_rate * db1 W2 -= learning_rate * dw2 B2 -= learning_rate * db2 W3 -= learning_rate * dw3 B3 -= learning_rate * db3 error.append(np.average(da2**2)) return error
def forward(nn, x): W1, W2, W3 = nn["W1"], nn["W2"], nn["W3"] b1, b2, b3 = nn["b1"], nn["b2"], nn["b3"] z1 = sigmoid(np.dot(x, W1) + b1) z2 = sigmoid(np.dot(z1, W2) + b2) z3 = np.dot(z2, W3) + b3 return z3
def forward(x, predict=True): a0 = x.T z1 = W1.dot(a0) + B1 a1 = sigmoid(z1) z2 = W2.dot(a1) + B2 a2 = sigmoid(z2) if predict is False: return a0, z1, a1, z2, a2 return a2
def forward_propagation(self, X): #forward propagation through our network self.z = np.dot( X, self.W1) # dot product of X (input) and first set of 14x9 weights self.z2 = sigmoid(self.z) # activation function self.z3 = np.dot( self.z2, self.W2 ) # dot product of hidden layer (z2) and second set of 9x1 weights output = sigmoid(self.z3) # final activation function return output
def forward(network, x): W1, W2, W3 = network['W1'], network['W2'], network['W3'] b1, b2, b3 = network['b1'], network['b2'], network['b3'] a1 = np.dot(x, W1) + b1 z1 = sigmoid(a1) a2 = np.dot(z1, W2) + b2 z2 = sigmoid(a2) a3 = np.dot(z2, W3) + b3 y = identity(a3) return y
def forward(network, x): Wait1, Wait2, Wait3 = network['Wait1'], network['Wait2'], network['Wait3'] bias1, bias2, bias3 = network['bias1'], network['bias2'], network['bias3'] a1 = np.dot(x, Wait1) + bias1 z1 = sigmoid(a1) a2 = np.dot(z1, Wait2) + bias2 z2 = sigmoid(a2) a3 = np.dot(z2, Wait3) + bias3 y = softmax(a3) return y
def forward(x, predict=True): a0 = x.T z1 = W1.dot(a0) + B1 a1 = sigmoid(z1) z2 = W2.dot(a1) + B2 a2 = sigmoid(z2) z3 = W3.dot(a2) + B3 a3 = sigmoid(z3) if predict is False: return a0, z1, a1, z2, a2, z3, a3 return a3
def forward_propagate(parameters,X, L,dropout=False): """ computes the forward propagation of the nerual network """ caches = {} caches["Z1"] = parameters["W1"].dot(X) + parameters["b1"] caches["a1"] = a.relu(caches["Z1"]) if dropout == True: caches["D1"] = np.random.rand(caches["a1"].shape[0],caches["a1"].shape[1]) < 0.8 caches["a1"] *= caches["D1"] caches["a1"] /= 0.5 caches["Z2"] = parameters["W2"].dot(caches["a1"]) + parameters["b2"] caches["a2"] = a.relu(caches["Z2"]) if dropout == True: caches["D2"] = np.random.rand(caches["a2"].shape[0],caches["a2"].shape[1]) < 0.8 caches["a2"] *= caches["D2"] caches["a2"] /= 0.5 # on the last layer we would like to compute the sigmoid for each examples caches["Z3"] = parameters["W3"].dot(caches["a2"]) + parameters["b3"] caches["a3"] = a.sigmoid(caches["Z3"]) return caches["a3"], caches
def activation_function(self, Z): if self.activation_function_name == 'relu': return relu(Z) elif self.activation_function_name == 'sigmoid': return sigmoid(Z) else: return tanh(Z)
def feedForward(self, inputs): if len(inputs) != self.input-1: raise ValueError('Wrong number of inputs') # input activations self.ai = np.append(inputs, [1]) # add bias node # hidden activations self.ah = sigmoid(self.ai.dot(self.wi)) # self.ah = relu(self.ai.dot(self.wi)) # output activations self.ao = sigmoid(self.ah.dot(self.wo)) # self.ao = relu(self.ah.dot(self.wo)) return softmax(self.ao)
def backpropagation(x, y, wo, wh): mses = [] while True: # x == > 515 * 8 net_h = np.dot(x, wh.T) # == > 515 * 10 net_h_active = sigmoid(net_h) # == > 515 * 10 net_o = np.dot(net_h_active, wo.T) # == > 515 * 1 net_o_active = net_o # == > 515 * 1 sigma = y - net_o_active # == > 515 * 1 error = sum_error(sigma) mses.append(error) if mses[-1] < 10: return mses, wo, wh sigma_o = np.dot(sigma.T, net_h) # == > 1 * 10 sigma_h = np.dot((np.dot(sigma, wo) * sigmoid_dash(net_h_active)).T, x) # == > 10 * 8 wo = wo + 0.0001 * sigma_o # == > 1 * 10 wh = wh + 0.0001 * sigma_h # == > 10 * 8
def activation_forward(self, A_prev, W, b, activation): if activation == 'sigmoid': Z, linear_cache = self.linear_forward(A_prev, W, b) A, activation_cache = sigmoid(Z) elif activation == "tanh": Z, linear_cache = self.linear_forward(A_prev, W, b) A, activation_cache = tanh(Z) elif activation == "relu": Z, linear_cache = self.linear_forward(A_prev, W, b) A, activation_cache = relu(Z) elif activation == "leaky_relu": Z, linear_cache = self.linear_forward(A_prev, W, b) A, activation_cache = leaky_relu(Z) else: print('no activation function') assert (A.shape == (W.shape[0], A_prev.shape[1])) cache = (linear_cache, activation_cache) return A, cache
def backprop(self, x, y): new_biase = [np.zeros(b.shape) for b in self.biases] new_weight = [np.zeros(w.shape) for w in self.weights] activation = x activations = [x] # list to store all the activations, layer by layer zs = [] # list to store all the z vectors, layer by layer for b, w in zip(self.biases, self.weights): z = np.dot(w, activation) + b zs.append(z) activation = sigmoid(z) activations.append(activation) # backward pass delta = self.cost_derivative(activations[-1], y) * sigmoid_prime( zs[-1]) new_biase[-1] = delta new_weight[-1] = np.dot(delta, activations[-2].transpose()) for l in range(2, self.num_layers): z = zs[-l] sp = sigmoid_prime(z) delta = np.dot(self.weights[-l + 1].transpose(), delta) * sp new_biase[-l] = delta new_weight[-l] = np.dot(delta, activations[-l - 1].transpose()) return (new_biase, new_weight)
def testName(self): # read data data = reader.read("../data/log_1_fixed.txt", jstartline=15000, maxlines=5000) # preprocess data preprocess.preproc(data) # initialize model layers = [13, 8, 1] activf = [activation.linear(), activation.tanh(), activation.sigmoid()] # , activation.tanh(1.75, 3./2.), net = ffnet.FFNet(layers, activf) net.initw(0.1) # create training options opts = trainb.options() # write function f = open("../output/trainb_test.txt", "w+") writefcn = lambda s: f.write(s) # training trainb.train(data, opts, net, writefcn) # close file f.close()
def forwardPropagate(self, inputs): outputs = [inputs] for weights in self.network: outputs[-1] = np.c_[outputs[-1], np.ones(len(outputs[-1]))] outputs.append(act.sigmoid(np.dot(outputs[-1], weights))) print('outputs', outputs, '\n') return outputs
def testName(self): # read data data = reader.read("../data/log_1_fixed.txt", jstartline=15000, maxlines=5000) # preprocess data preprocess.preproc(data) # initialize model layers = [13, 8, 1] activf = [ activation.linear(), activation.tanh(), activation.sigmoid() ] # activation.tanh(1.75, 3./2.), net = ffnet.FFNet(layers, activf) net.initw(0.1) # create training options opts = trainsg.options() # write function f = open("../output/trainsg_test.txt", "w+") writefcn = lambda s: f.write(s) # training trainsg.train(data, opts, net, writefcn) # close file f.close()
def train(x, y, iterations=50000, learning_rate=0.001): global W1, W2, B1, B2, error m = x.shape[0] error = [] for _ in range(iterations): a0, z1, a1, z2, a2 = forward(x, predict=False) da2 = a2 - y.T dz2 = da2 * linear(z2, derivative=True) dw2 = dz2.dot(a1.T) / m db2 = np.sum(dz2, axis=1, keepdims=True) / m da1 = W2.T.dot(dz2) dz1 = np.multiply(da1, sigmoid(z1, derivative=True)) dw1 = dz1.dot(a0.T) / m db1 = np.sum(dz1, axis=1, keepdims=True) / m W1 -= learning_rate * dw1 B1 -= learning_rate * db1 W2 -= learning_rate * dw2 B2 -= learning_rate * db2 error.append(np.average(da2**2)) return error
def _forward(z: np.array, W: np.array, b: np.array, activation: str) -> np.array: """Propagate the signal from the previous to the next layer.""" a = np.dot(z, W) + b if activation == 'sigmoid': return sigmoid(a) elif activation == 'identity': return identity(a)
def backPropagate(self, inputs, predicted, log=True): outputs = self.forwardPropagate(inputs) errors = [predicted - outputs[-1]] print(errors[0]) deltas = [ act.sigmoid(outputs[-1], d=True) * errors[-1] * self.learningRate ] print(deltas[0]) for i in range(len(self.network)): print() errors.insert(0, deltas[0].dot(self.network[1].T)) print(errors[0]) deltas.insert(0, errors[0] * act.sigmoid(outputs[i + 1], d=True)) print(deltas[0]) for i in range(len(deltas)): self.network[i] += deltas[i] * outputs[i]
def calc_output(self): """ 根据式1计算节点的输出 """ output = reduce( lambda ret, conn: ret + conn.upstream_node.output * conn.weight, self.upstream, 0) self.output = sigmoid(output)
def linear_activation_forward(A_prev, W, b, activation): Z = np.dot(W, A_prev) + b if activation == "sigmoid": A = sigmoid(Z) elif activation == "relu": A = relu(Z) elif activation == "tanh": A = tanh(Z) return A, Z
def test_sigmoid_deriv(): """Test sigmoid activation function derivative""" x = np.array([[0, 1, 3], [-1, 0, -5], [1, 0, 3], [10, -9, -7]]) y = np.array([[0.25, 0.19661, 0.04518], [0.19661, 0.25, 0.00665], [0.19661, 0.25, 0.04518], [0.00005, 0.00012, 0.00091]]) assert np.allclose(sigmoid(x, deriv=True), y, atol=0.00001)
def test_sigmoid(): """Test sigmoid activation function""" x = np.array([[0, 1, 3], [-1, 0, -5], [1, 0, 3], [10, -9, -7]]) y = np.array([[0.5, 0.73106, 0.95257], [0.26894, 0.5, 0.00669], [0.73106, 0.5, 0.95257], [0.99995, 0.00012, 0.00091]]) assert np.allclose(sigmoid(x), y, atol=0.00001)
def predict(self, x): W1, W2 = self.params['W1'], self.params['W2'] b1, b2 = self.params['b1'], self.params['b2'] a1 = np.dot(x, W1) + b1 z1 = sigmoid(a1) a2 = np.dot(z1, W2) + b2 y = softmax(a2) return y
def test_sigmoid(self): rtol = 1e-5 size = 10 for _ in range(1000): x = np.random.uniform(low=-1000., high=1000., size=size).flatten() test_buffer = list_2_swig_float_pointer(x.tolist(), size) y_numpy = np.array(tf.keras.activations.sigmoid(x)).tolist() y_nn4mc = activation.sigmoid(test_buffer.cast(), size) y_nn4mc = swig_py_object_2_list(y_nn4mc, size) assert np.allclose(y_nn4mc, y_numpy, rtol) print("sigmoid passed!")
def test_sigmoid(self): self.assertEqual( list(a.sigmoid([1, 2, 3, 4, 5])), [ 0.7310585786300049, 0.8807970779778823, 0.9525741268224334, 0.9820137900379085, 0.9933071490757153 ] )
def step(z_t, zi_t, zf_t, zo_t, c_tm1, h_tm1): # new information Z_t = tanh(z_t + T.dot(h_tm1, U)) # input gate Zi_t = sigmoid(zi_t + T.dot(h_tm1, Ui) + T.dot(c_tm1, Vi)) # forget gate Zf_t = sigmoid(zf_t + T.dot(h_tm1, Uf) + T.dot(c_tm1, Vf)) # new plus old/unforgetten memory c_t = Z_t * Zi_t + c_tm1 * Zf_t # output gate Zo_t = sigmoid(zo_t + T.dot(h_tm1, Uo) + T.dot(c_t, Vo)) # output information h_t = tanh(c_t) * Zo_t return c_t, h_t
def test_apply(self): # create a simple network net = ffnet.FFNet( [k, q, m], [activation.linear(), activation.tanh(), activation.sigmoid()]) # some input x = [1] * k net.apply(x)
def forwardPropagation(self, row, count): listInputs = row for layer in self.network: listHidLayerInputs = [] for neuron in layer: activation = self.activate(neuron, listInputs) + self.bias[count] output = sigmoid(activation) listHidLayerInputs.append(output) listInputs = listHidLayerInputs return listInputs
def backprop(Theta1, Theta2, X, y): N = X.shape[0] K = Theta2.shape[0] J = 0 Delta2 = np.zeros(Theta2.shape) Delta1 = np.zeros(Theta1.shape) for i in range(N): # Forward propagation, saving intermediate results a1 = np.concatenate(([1], X[i])) # Input layer z2 = np.dot(Theta1, a1) a2 = np.concatenate(([1], sigmoid(z2))) # Hidden Layer z3 = np.dot(Theta2, a2) a3 = sigmoid(z3) # Output layer y0 = one_hot(K, y[i]) # Cross entropy J -= np.dot(y0.T, np.log(a3))+np.dot((1-y0).T, np.log(1-a3)) # Calculate the weight deltas delta_3 = a3-y0 delta_2 = np.dot(Theta2.T, delta_3)[1:]*sigmoidGradient(z2) Delta2 += np.outer(delta_3, a2) Delta1 += np.outer(delta_2, a1) J /= N Theta1[:, 0] = np.zeros(Theta1.shape[0]) Theta2[:, 0] = np.zeros(Theta2.shape[0]) Theta1_grad = Delta1/N Theta2_grad = Delta2/N return [J, Theta1_grad, Theta2_grad]
def get_precision(weights, bias, test_data): ''' 通过输入的权重和偏置得到准确率,测试集为test_data ''' test_alpha = Normalize(test_data[0]) for w, b in zip(weights, bias): #print(test_alpha.shape,w.shape) test_alpha = sigmoid(np.dot(test_alpha, w) + b.T) test_result = np.array( [np.argmax(test_alpha[i, :]) for i in range(len(test_alpha))]) #统计预测结果 return sum(int(i == j) for (i, j) in zip(test_result, test_data[1])) / len( test_data[1])
def test_sigmoid(self): sigm = activation.sigmoid(-2.3, 10.4) x = 0.2; dx = 1.e-7; f = sigm.f(x) df_a = sigm.df(f) df_n = (sigm.f(x + dx) - sigm.f(x - dx)) / 2 / dx print "Numerical: %e, Analytical: %e" % (df_a, df_n) self.assertAlmostEqual(df_a, df_n, 8, "Numerical derivative is not equal to analytical")
def train(k, data): # initialize new model layers = [13, 8, 1] activf = [activation.linear(), activation.tanh(), activation.sigmoid()] net = ffnet.FFNet(layers, activf) net.initw(0.1) # use default training options opts = trainsg.options() opts.rate = 2.e-4 # write function f = open("../output/train-%s.txt" % k, "w+") writefcn = lambda s: f.write(s) # training net = trainsg.train(data, opts, net, writefcn) # close file f.close() # return trained network return net
def forward(self, prev_layer): if prev_layer.layer_property == Layer_Property['conv']: self.neurons = act.sigmoid(self.weight.dot(prev_layer.neurons.flatten())) else: self.neurons = act.sigmoid(self.weight.dot(prev_layer.neurons))
def forward(self, prev_layer): self.neurons = act.sigmoid(self.weight.dot(prev_layer.neurons))
def forward(self, prev_layer): prev_matrix = prev_layer.neurons for cur_feature in range(self.map_num): for prev_feature in range(prev_layer.map_num): self.neurons[cur_feature] += signal.convolve2d(prev_matrix[prev_feature], np.rot90(self.conv_filter[prev_feature][cur_feature], 2), mode='valid') self.neurons[cur_feature] = act.sigmoid(self.neurons[cur_feature])