def stocGradAscent1(dataMatrix,classLabels,numIter=150):
m,n=shape(dataMatrix)
dataIndex=range(m)
weights=ones(n)
for j in range(numIter):
for i in range(m):
alpha=4/(1.0+j+i)+0.01
randIndex=int(random.uniform(0,len(dataIndex)))
h=sigmoid(sum(dataMatrix[randIndex]*weights))
error=classLabels[randIndex]-h
weights=weights+alpha*error*dataMatrix[randIndex]
del(dataIndex[randIndex])
return weights
def __init__(self):
self.layers = []
self.history = {"loss": []}
self.cost = None
self.activation_funcs = {
"relu": activation.relu(),
"softmax": activation.softmax(),
"sigmoid": activation.sigmoid(),
"linear": activation.identity(),
"tanh": activation.tanh(),
"swish": activation.swish(),
"lrelu": activation.lrelu()
}
self.cost_funcs = {
"squared loss": error.SquaredError(),
"cross entropy": error.CrossEntropy()
}
self.layer_types = {
"dense": layers.Dense,
}
from data_prep import process_csv
from nn.model import NeuralNetwork
from nn.layers import InputLayer, Dense
from nn.loss import CrossEntropy
from nn.optimizer import SGD
from nn.activations import sigmoid, tanh
test_file = '../data/mnist_test.csv'
train_file = '../data/mnist_train.csv'
x_train, y_train = process_csv(train_file)
x_test, y_test = process_csv(test_file)
model = NeuralNetwork()
model.addLayer(InputLayer((1, 784)))
model.addLayer(Dense(neuron_count=300, activation=tanh()))
model.addLayer(Dense(neuron_count=10, activation=sigmoid()))
model.compile(loss=CrossEntropy(), optimizer=SGD(alpha=0.000006))
train_loss, train_acc, val_loss, val_acc = model.fit(x_test,
y_test,
validation_set=True,
validation_split=0.1,
epochs=1,
batch_size=100)
def lstm_cell_forward(Xt, h_prev, c_prev, parameters):
"""
Input:
- Xt: Input data at timestep "t", shape: (N, D)
: N : #of samples.
: D : #of input examples. D = 28 in MNIST dataset
- h_prev: Hidden state at timestep "t-1", shape: (N, H)
: N : #of samples.
: H : #of hidden neurans
- c_prev: Memory state at timestep "t-1", shape: (N,H)
- parameters: a dictionary containing:
: Wf : Weight matrix of the forget gate, shape (H+D, H)
: Wi : Weight matrix of the update gate, shape (H+D, H)
: Wo : Weight matrix of the output gate, shape (H+D, H)
: Wc : Weight matrix of the first "tanh", shape (H+D, H)
: Wy : Weight matrix relating the hidden-state to the output, shape (H, M), M = 10 in MNIST dataset
: bf : Bias, shape (1, H)
: bi : Bias, shape (1, H)
: bo : Bias, shape (1, H)
: bc : Bias, shape (1, H)
: by : Bias, shape (1, M)
Returns:
- h_next : next hidden state, shape (N, H)
- c_next : next memory state, shape (N, H)
- yt_pred: prediction at timestep "t", shape (N, M)
- cache : tuple of values needed for the backward pass,
contains (h_next, c_next, h_prev, c_prev, Xt, parameters)
Note:
ft/it/ot stand for the forget/update/output gates, cct stands for the candidate value (c tilde), c stands for the memory value
"""
# Retrieve parameters from "parameters"
Wf = parameters["Wf"]
Wi = parameters["Wi"]
Wo = parameters["Wo"]
Wc = parameters["Wc"]
Wy = parameters["Wy"]
bf = parameters["bf"]
bi = parameters["bi"]
bo = parameters["bo"]
bc = parameters["bc"]
by = parameters["by"]
# Retrieve dimensions from shapes of Xt and Wy
N, D = Xt.shape
H, M = Wy.shape
# Concatenate h_prev and Xt
concat = np.zeros((N, H+D))
concat[:, :H] = h_prev
concat[:, H:] = Xt
# Compute values for ft, it, cct, c_next, ot, h_next
ft = sigmoid(np.dot(concat, Wf) + bf)
it = sigmoid(np.dot(concat, Wi) + bi)
ot = sigmoid(np.dot(concat, Wo) + bo)
cct = np.tanh(np.dot(concat, Wc) + bc)
c_next = ft * c_prev + it * cct
h_next = ot * np.tanh(c_next)
# Compute prediction of the LSTM cell
yt_pred = softmax(np.dot(h_next, Wy) + by)
# store values needed for backward propagation in cache
cache = (h_next, c_next, h_prev, c_prev, ft, it, cct, ot, Xt, parameters)
return h_next, c_next, yt_pred, cache
def classifyVector(intX,weights):
prob=sigmoid(sum(intX*weights))
if prob>0.5:return 1.0
else:return 0.0