def test(self):
np.random.shuffle(self._test_data)
test_X = self._test_data[:, :15]
Y = self._test_data[:, -1]
Y_real = []
for y in Y:
Y_real.append([y])
Y_real = np.asarray(Y_real)
inputLayer = test_X
sum_sinapse0 = np.dot(inputLayer, self._weights0)
hiddenLayer = libfunc.sigmoid(sum_sinapse0)
sum_sinapse1 = np.dot(hiddenLayer, self._weights1)
outputLayer = libfunc.sigmoid(sum_sinapse1)
plt.clf()
plt.plot(Y)
plt.plot(outputLayer)
plt.legend(['Real', 'Estimado'], loc='upper right')
plt.grid()
plt.savefig(
os.path.dirname(__file__) + "\\graphs\\test\\" + "test_" +
str(self._epochs) + str(self._tent) + ".png")
# plt.show()
return self.nash_sutcliffe(Y_real, outputLayer)
def _forward_propagation(self, x):
# layer input -> hidden
z1 = np.dot(x, self.W_xh) + self.b_h
a1 = af.sigmoid(z1)
# layer hidden -> output
z2 = np.dot(a1, self.W_hy) + self.b_y
y_hat = af.sigmoid(z2)
return z1, a1, z2, y_hat
def _feedforward(self, a):
"""
calculate the output of the network
:param a: a numpy ndarray (n, 1)
:return: a single number corresponding to the output of the network
"""
for b, w in zip(self.biases, self.weights):
# To compensate for the increase in number of neurons activated
# we divide the weights and biases by the dropout size.
if self.dropout_enabled:
a = af.sigmoid(
np.dot(w / self.dropout_size, a) + b / self.dropout_size)
else:
a = af.sigmoid(np.dot(w, a) + b)
return a
def linear_activation_forward(A_prev, W, b, activation, output_size):
# """
# Implement the forward propagation for the LINEAR->ACTIVATION layer
# Arguments:
# A_prev -- activations from previous layer (or input data): (size of previous layer, number of examples)
# W -- weights matrix: numpy array of shape (size of current layer, size of previous layer)
# b -- bias vector, numpy array of shape (size of the current layer, 1)
# activation -- the activation to be used in this layer, stored as a text string: "sigmoid" or "relu"
# output_size - bit width of output data
# Returns:
# A -- the output of the activation function, also called the post-activation value
# cache -- a python tuple containing "linear_cache" and "activation_cache";
# stored for computing the backward pass efficiently
# """
if activation == "sigmoid":
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = sigmoid(Z, output_size)
elif activation == "relu":
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = relu(Z, output_size)
assert (A.shape == (W.shape[0], A_prev.shape[1]))
cache = (linear_cache, activation_cache)
return A, cache
def linear_activation_forward(a_prev,
w,
b,
activation="sigmoid",
keep_prob=1.0):
if activation == "sigmoid":
z, linear_cache = linear_forward(a_prev, w, b)
a, activation_cache = sigmoid(z)
elif activation == "ReLu":
z, linear_cache = linear_forward(a_prev, w, b)
a, activation_cache = ReLu(z)
elif activation == "leaky_ReLu":
z, linear_cache = linear_forward(a_prev, w, b)
a, activation_cache = leaky_ReLu(z)
# Dropout Regularization
d = np.random.rand(a.shape[0], a.shape[1])
d = np.where(d < keep_prob, 1,
0) # if keep_prob = 1 - no effect of dropout!
a = np.multiply(a, d)
assert (a.shape == (w.shape[0], a_prev.shape[1]))
#cache of layer l contains ((a[l-1] , w[l] , b[l]) , z[l])
cache = (linear_cache, activation_cache, d)
return a, cache
def predict(features, weights): ''' Returns 1D array of probabilities that the class label == 1 ''' z = np.dot(features, weights) return sigmoid(z)
def example(game, n=5):
"""
train neural network
:param game: val_fun -> list of data of game [{'x':[...], 'y': [...]} ... ]
:return:
"""
shape = [2, 4, 1]
act_fun = sigmoid()
fit_kwargs = {}
max_epoch = 10**3
model = FeedForwardNeuralNet(shape=shape, act_fun=act_fun)
model.settings(name='model')
model.load()
algorithm = NAG()
algorithm.compile(model=model)
algorithm.settings(max_epoch=max_epoch, check_loss_freq=0)
for _ in range(n):
training_data = game(val_fun=model)
algorithm.compile(data=training_data)
algorithm.fit(**fit_kwargs)
model.save()
def process_hidden_layers(cache, weights_biases, num_layers):
for layer in range(1, num_layers + 1):
Z = np.dot(weights_biases['W' + str(layer)],
cache['A' + str(layer - 1)])
A = af.sigmoid(Z)
cache['Z' + str(layer)] = Z
cache['A' + str(layer)] = A
def activation_forward(A_prev, W, b, activation_way):
"""
:param A_prev:
:param W:
:param b:
:param activation_way: -- a text string indicate the way we activate this layer, "sigmoid","relu",...
:return:
A -- the activation output of this layer
cache -- a dictionary contains "linear_cache' and "activation_cache"
"""
cache = dict()
if activation_way == "relu":
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = relu(Z)
elif activation_way == "sigmoid":
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = sigmoid(Z)
elif activation_way == "tanh":
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = tanh(Z)
elif activation_way == "softmax":
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = softmax(Z)
cache["linear_cache"] = linear_cache
cache["activation_cache"] = activation_cache
return A, cache
def model(self):
self.N, self.D = self.X.shape
self.N_t, self.D_t = self.Xtest.shape
ones = np.array([[1] * self.N]).T
ones_t = np.array([[1] * self.N_t]).T
self.Xb = np.concatenate((ones, self.X), axis=1)
self.X_t_b = np.concatenate((ones_t, self.Xtest), axis=1)
self.w = np.random.randn(self.D + 1)
z = self.Xb.dot(self.w)
self.Yhat = sigmoid(z)
def test(self):
test_X = self._test_data[:, :15]
Y = self._test_data[:, -1]
inputLayer = test_X
sum_sinapse0 = np.dot(inputLayer, self._weights0)
hiddenLayer = libfunc.sigmoid(sum_sinapse0)
sum_sinapse1 = np.dot(hiddenLayer, self._weights1)
outputLayer = libfunc.sigmoid(sum_sinapse1)
print()
plt.clf()
plt.plot(Y)
plt.plot(outputLayer)
plt.legend(['Observed', 'Predict'], loc='upper right')
plt.grid()
plt.savefig(
os.path.dirname(__file__) + "\\graphs\\test\\" + "test_" +
str(self._epochs) + ".png")
def train(self):
for i in range(self.epochs * 100):
if i % 100 == 0:
print(cross_entropy(self.Y, self.Yhat))
# update weights
self.w += self.lr * (np.dot((self.Y - self.Yhat).T, self.Xb) -
l2_regularization(self.l, self.w, True))
self.Yhat = sigmoid(self.Xb.dot(self.w))
print('Final w:', self.w)
def validation(self):
np.random.shuffle(self._valid_data)
valid_X = self._valid_data[:, :7]
Y = self._valid_data[:, -1]
Y_real = []
for y in Y:
Y_real.append([y])
Y_real = np.asarray(Y_real)
inputLayer = valid_X
sum_sinapse0 = np.dot(inputLayer, self._bestweights0)
hiddenLayer = libfunc.sigmoid(sum_sinapse0) * self._bias
sum_sinapse1 = np.dot(hiddenLayer, self._bestweights1)
outputLayer = libfunc.sigmoid(sum_sinapse1) * self._bias
rmse = np.average((Y_real - outputLayer)**2)
return self.nash_sutcliffe(
Y_real, outputLayer), rmse, self._bestweights0, self._bestweights1
def test(self, weights0, weights1):
np.random.shuffle(self._test_data)
test_X = self._test_data[:, :7]
Y = self._test_data[:, -1]
Y_real = []
for y in Y:
Y_real.append([y])
Y_real = np.asarray(Y_real)
inputLayer = test_X
sum_sinapse0 = np.dot(inputLayer, weights0)
hiddenLayer = libfunc.sigmoid(sum_sinapse0) * self._bias
sum_sinapse1 = np.dot(hiddenLayer, weights1)
outputLayer = libfunc.sigmoid(sum_sinapse1) * self._bias
plt.clf()
plt.plot(Y_real)
plt.plot(outputLayer)
plt.legend(['Real', 'Estimado'], loc='upper right')
plt.grid()
plt.savefig(
os.path.dirname(__file__) + "\\graphs\\test\\" + "test.png")
# plt.show()
rmse = np.average((Y_real - outputLayer)**2)
plt.clf()
plt.plot((Y_real - outputLayer))
plt.grid()
plt.savefig(
os.path.dirname(__file__) + "\\graphs\\test\\" +
"test_erro_ep.png")
print("NASH : " + str(self.nash_sutcliffe(Y_real, outputLayer)) +
" - MSE: " + str(rmse))
def classify(self, dataVector): self.inputLayer = [] self.hiddenLayer = [] self.outputLayer = [] self.inputLayer.append(dataVector) self.inputLayer[0].insert(0, 1.0) #multiple inputlayer and weights z1 = np.matrix(self.inputLayer) * np.matrix(self.weightMatrices[0]) z1 = z1.tolist() #print z1 #get hidden layer t = [] for i in z1[0]: t.append(activefunc.sigmoid(i)) self.hiddenLayer.append(t) #print self.hiddenLayer self.hiddenLayer[0].insert(0, 1.0) #print self.hiddenLayer #print len(self.hiddenLayer) #print len(self.hiddenLayer[0]) #get output layer z2 = np.matrix(self.hiddenLayer) * np.matrix(self.weightMatrices[1]) z2 = z2.tolist() #print z2 #get outputlayer t = [] for i in z2[0]: t.append(activefunc.sigmoid(i)) self.outputLayer.append(t) for i in range(len(self.outputLayer[0])): if self.outputLayer[0][i] == max(self.outputLayer[0]): self.outputLayer[0][i] = 1 else: self.outputLayer[0][i] = 0 return self.outputLayer
def middle_layer(x, w, b):
"""middle layer
多層パーセプトロンの中間層
Args:
x: input
w: wight
b: bias
Retuens:
output of neuron
"""
u = np.dot(x, w) + b
return sigmoid(u)
def linear_activation_forward(A_prev, W, b, activation):
if activation == 'sigmoid':
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = sigmoid(Z)
elif activation == 'relu':
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = relu(Z)
assert A.shape == (W.shape[0], A_prev.shape[1])
cache = (linear_cache, activation_cache)
return A, cache
def test(x, y, w1, w2):
predictions = []
scores = 0
for i in range(len(x)):
l0 = x[i]
l1 = LeakyReLU(l0.dot(w1))
l2 = sigmoid(l1.dot(w2))
predictions.append(
[1, 0]) if l2[0] > l2[-1] else predictions.append([0, 1])
if predictions[i][0] == y[i][0] and predictions[i][1] == y[i][1]:
scores += 1
return scores / len(y)
def linear_activation_forward(A_prev, W, b, activation):
if activation == "sigmoid":
# Inputs: "A_prev, W, b". Outputs: "A, activation_cache".
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = sigmoid(Z)
elif activation == "relu":
# Inputs: "A_prev, W, b". Outputs: "A, activation_cache".
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = relu(Z)
assert (A.shape == (W.shape[0], A_prev.shape[1]))
cache = (linear_cache, activation_cache)
return A, cache
def __feedforward(self):
activation = self.__activation_func()[0]
bias1 = np.ones((self.input.shape[0], 1))
self.z1 = np.concatenate((np.dot(self.input, self.weights1), bias1),
axis=1)
self.layer1 = activation(self.z1)
self.layers = [self.layer1]
self.zs = [self.z1]
for weight in self.weights:
prev_layer = self.layers[-1]
bias = np.ones((prev_layer.shape[0], 1))
z = np.concatenate((np.dot(prev_layer, weight), bias), axis=1)
layer = activation(z)
self.layers.append(layer)
self.zs.append(z)
self.z_last = np.dot(self.layers[-1], self.last_weights)
self.output = self.z_last if self.objective == 'regression' else sigmoid(
self.z_last)
return self.output
def _forwardprop(self, inputs):
"""
:param inputs:
a matrix X = [x1, ... , xm], where xi is a single training input, 784*1
:return:
(A, Z), where A is a list of matrices of activations of each training input;
Z is the list of matrices of intermediate z values of each training input
"""
activation = inputs
activations = [activation]
zs = []
for w, b, r in zip(self.weights, self.biases, self.mask):
z = np.dot(w, activation) + b
zs.append(z)
activation = r * af.sigmoid(z)
activations.append(activation)
return (activations, zs)
def forwardprop_LogReg(X, parameters):
"""
Argument:
X -- matrix of shape (n_x, m) of inputs, each column is a training example
parameters -- dictionary containing the parameters:
W -- weight vector of shape (n_x, 1)
b -- bias float
Returns:
A -- an array of shape (1, m) containing the outputs for each example
"""
W = parameters['W']
b = parameters['b']
Z = np.dot(W.T, X) + b
A = sigmoid(Z)
return A
def __run_network(self,
registers: np.array,
debug: dict = None) -> np.array:
def take_params(values, i):
"""Return the next pair of weights and biases after the
starting index and the new starting index."""
return values[i], values[i + 1], i + 2
# Extract the 0th (i.e. P( x = 0 )) component from all registers.
last_hidden_layer = np.array(registers[:, 0][None, ...],
dtype=np.float32)
# Propogate forward to hidden layers.
idx = 0
for i in range(self.context.num_hidden_layers):
W, b, idx = take_params(self.context.network, idx)
last_hidden_layer = relu(last_hidden_layer.dot(W) + b)
controller_coefficients = []
for i, gate in enumerate(self.context.gates):
coeffs = []
for j in range(gate.arity):
W, b, idx = take_params(self.context.network, idx)
coeff = softmax(last_hidden_layer.dot(W) + b)
coeffs.append(coeff)
controller_coefficients.append(coeffs)
# Forward propogate to new register value coefficients.
for i in range(self.context.num_regs):
W, b, idx = take_params(self.context.network, idx)
coeff = softmax(last_hidden_layer.dot(W) + b)
controller_coefficients.append(coeff)
# Forward propogate to generate willingness to complete.
W, b, idx = take_params(self.context.network, idx)
complete = sigmoid(last_hidden_layer.dot(W) + b)
if debug is not None:
debug.fi = np.around(complete.sum(), 3)
return controller_coefficients, complete
def feed_forward(self, x):
if self.activation_function == 'sigmoid':
# z = weighted sum
z = np.dot(self.weights, x) + self.bias
# a = activation function
a = af.sigmoid(z)
# y = output
y = a
elif self.activation_function == 'heavyside':
# z = weighted sum
z = np.dot(self.weights, x) + self.bias
# a = activation function
a = af.heavyside(z)
# y = output
y = a
elif self.activation_function == 'relu':
# z = weighted sum
z = np.dot(self.weights, x) + self.bias
# a = activation function
a = af.relu(z)
# y = output
y = a
elif self.activation_function == 'leaky_relu':
# z = weighted sum
z = np.dot(self.weights, x) + self.bias
# a = activation function
a = af.leaky_relu(z)
# y = output
y = a
elif self.activation_function == 'tanh':
# z = weighted sum
z = np.dot(self.weights, x) + self.bias
# a = activation function
a = af.tanh(z)
# y = output
y = a
return y, z
def activationFunction(self, data): return af.sigmoid(data)
def train(self): #generate random weights self.generateWeights(self.weightMatrices, neuronsPerHiddenLayer - 1, neuronsInputLayer) self.generateWeights(self.weightMatrices, neuronsOutputLayer, neuronsPerHiddenLayer) #print self.weightMatrices[0] iterator = 0 oldCost = 0.0 cost = 0.0 learningRatio = maxLearingRatio while iterator < maxLoop: #loop for each vector for vector in self.trainSet: #forward propagation self.inputLayer.append(vector[1:]) self.inputLayer[0].insert(0, 1.0) #print self.inputLayer #multiple inputlayer and weights z1 = np.matrix(self.inputLayer) * np.matrix(self.weightMatrices[0]) z1 = z1.tolist() #print z1 #get hidden layer t = [] for i in z1[0]: t.append(activefunc.sigmoid(i)) self.hiddenLayer.append(t) #print self.hiddenLayer self.hiddenLayer[0].insert(0, 1.0) #print self.hiddenLayer #print len(self.hiddenLayer) #print len(self.hiddenLayer[0]) #get output layer z2 = np.matrix(self.hiddenLayer) * np.matrix(self.weightMatrices[1]) z2 = z2.tolist() #print z2 #get outputlayer t = [] for i in z2[0]: t.append(activefunc.sigmoid(i)) self.outputLayer.append(t) #print self.outputLayer #get cost function cost = self.costFunction(self.outputLayer) #print cost if np.abs(cost - oldCost) <= limit: break #print np.abs(cost - oldCost) #print cost #backward propagation delta = [] t1 = np.zeros((neuronsInLayers[0], neuronsInLayers[1] - 1)).tolist() t2 = np.zeros((neuronsInLayers[1], neuronsInLayers[2])).tolist() delta.append(t1) delta.append(t2) #print delta #get error value for output layer sigma3 = [] t = [] for i in range(neuronsInLayers[2]): t.append(self.outputLayer[0][i] - vector[0][i]) sigma3.append(t) #print sigma3 #get error value for hidden layer sigma2 = [] sigma2 = np.matrix(sigma3) * np.matrix(np.transpose(self.weightMatrices[1])) sigma2 = sigma2.tolist() #print sigma2 derivative = [] t = [] for i in self.hiddenLayer[0]: t.append(activefunc.derivativeSigmoid(i)) derivative.append(t) #print derivative t = [] for a,b in zip(sigma2[0], derivative[0]): t.append(a * b) sigma2 = [] sigma2.append(t[1:]) #print sigma2 #calculate delta[0] for i in range(len(delta[0])): for j in range(len(delta[0][i])): delta[0][i][j] += self.inputLayer[0][i] * sigma2[0][j] #print delta[0] #calculate delta[1] for i in range(len(delta[1])): for j in range(len(delta[1][i])): delta[1][i][j] += self.hiddenLayer[0][i] * sigma3[0][j] #print delta[1] #derivative for cost function costDerivative = [] costDerivative.append(self.derivativeCostFunction(delta, 0)) costDerivative.append(self.derivativeCostFunction(delta, 1)) #print costDerivative[0] #update weights for k in range(len(self.weightMatrices)): for i in range(len(self.weightMatrices[k])): for j in range(len(self.weightMatrices[k][i])): self.weightMatrices[k][i][j] = self.weightMatrices[k][i][j] - learningRatio * costDerivative[k][i][j] #print len(self.weightMatrices[0]) self.inputLayer = [] self.hiddenLayer = [] self.outputLayer = [] oldCost = cost learningRatio = learningRatio / (1.0 + (iterator / maxLoop)) if np.abs(cost - oldCost) <= limit: break iterator += 1 print "Error: %f" %cost
def mean_field_from_activation(self, vmap):
return activation_functions.sigmoid(vmap[self] - vmap[self.flipped_units])
def success_probability_from_activation(self, vmap):
return activation_functions.sigmoid(vmap[self])
def sample_from_activation(self, vmap):
p = activation_functions.sigmoid(vmap[self] - vmap[self.flipped_units])
return samplers.bernoulli(p)
def mean_field_from_activation(self, vmap):
return activation_functions.sigmoid(vmap[self])
def sample_from_activation(self, vmap):
p = activation_functions.sigmoid(vmap[self] - vmap[self.flipped_units])
return samplers.bernoulli(p)
def success_probability_from_activation(self, vmap):
return activation_functions.sigmoid(vmap[self])
def train(self):
rmseArray = []
epoch = self._epochs
# condicao de parada por epocas
while epoch > 0:
np.random.shuffle(self._train_data)
train_X = self._train_data[:, :15]
Y = self._train_data[:, -1]
train_Y = []
for y in Y:
train_Y.append([y])
train_Y = np.asarray(train_Y)
print("Epoch: " + str(epoch))
inputLayer = train_X
# Camada de entrada
sum_sinapse0 = np.dot(inputLayer, self._weights0)
hiddenLayer = libfunc.sigmoid(sum_sinapse0)
sum_sinapse1 = np.dot(hiddenLayer, self._weights1)
outputLayer = libfunc.sigmoid(sum_sinapse1)
outputLayerError = train_Y - outputLayer
# Condição
rmse = mean_squared_error(train_Y, outputLayer)
rmseArray.append(np.mean(outputLayerError))
if rmse < self._rmse_min and self._stop_params == "mse":
break
derivedOutput = deriFunc.derived_sigmoid(outputLayer)
deltaOutput = outputLayerError * derivedOutput
weight1T = self._weights1.T
deltaOutXWeight = deltaOutput.dot(weight1T)
deltaOutputHidden = deltaOutXWeight * deriFunc.derived_sigmoid(
hiddenLayer)
hiddenLayerT = hiddenLayer.T
weight1_new = hiddenLayerT.dot(deltaOutput)
self._weights1 = (self._weights1 * self._momentum) + (
weight1_new * self._learning_rate)
inputLayerT = inputLayer.T
weight0_new = inputLayerT.dot(deltaOutputHidden)
self._weights0 = (self._weights0 * self._momentum) + (
weight0_new * self._learning_rate)
# implementar validacao cruzada
cross_stop = False
if cross_stop and self._stop_params == "crossvalidation":
break
epoch -= 1
# fim de todas epocas
np.savetxt("RMSE.txt", rmseArray, fmt='%1.6f')
plt.clf()
plt.plot(rmseArray)
plt.grid()
plt.show()
def mean_field_from_activation(self, vmap):
return activation_functions.sigmoid(vmap[self] -
vmap[self.flipped_units])
def derived_sigmoid(x):
return func.sigmoid(x) * (1 - func.sigmoid(x))
def test_sigmoid(self):
result = af.sigmoid(np.array([[1, 2], [3, 4]]))
expected_result = np.array([[0.731058, 0.880797], [0.952574, 0.982013]])
difference = result - expected_result
self.assertTrue(np.linalg.norm(difference) < 1e-4)
def mean_field_from_activation(self, vmap):
return activation_functions.sigmoid(vmap[self])
