def rnn_cell_forward(Xt,h_prev,parameters):
'''
RNN Cell:
Input:
- Xt: (N,D) N=2000 D=28
- h_prev: (N,H) #of neurons in the hidden state. "prev" is actually for timestep "t-1"
- parameters:
: Wx: Weight matrix multiplying the input Xt, (D, H)
: Wh: Weight matrix multiplying the hidden state (H,H)
: Wy: Weight matrix relating to the hidden-state. Shape is (H,M) # M = 10
: bh: Bias, (1, H)
: by: Bias, (1, M)
Returns:
- h_next: next hidden state (N, H)
- yt_pred: prediction at timestep t, (N, M)
- cache : tuple of values needed for the back-propagation part, has shape (h_next, h_prev, Xt, parameters)
'''
Wx = parameters["Wx"]
Wh = parameters["Wh"]
Wy = parameters["Wy"]
bh = parameters["bh"]
by = parameters["by"]
# compute next activation state using the formula tanh(xxxx)
h_next = tanh(np.dot(Xt,Wx) + np.dot(h_prev,Wh) + bh)
yt_pred = softmax(np.dot(h_next, Wy) + by)
cache = (h_next, h_prev, Xt, parameters)
return h_next, yt_pred, cache
def __init__(self):
self.layers = []
self.history = {"loss": []}
self.cost = None
self.activation_funcs = {
"relu": activation.relu(),
"softmax": activation.softmax(),
"sigmoid": activation.sigmoid(),
"linear": activation.identity(),
"tanh": activation.tanh(),
"swish": activation.swish(),
"lrelu": activation.lrelu()
}
self.cost_funcs = {
"squared loss": error.SquaredError(),
"cross entropy": error.CrossEntropy()
}
self.layer_types = {
"dense": layers.Dense,
}
from data_prep import process_csv
from nn.model import NeuralNetwork
from nn.layers import InputLayer, Dense
from nn.loss import CrossEntropy
from nn.optimizer import SGD
from nn.activations import sigmoid, tanh
test_file = '../data/mnist_test.csv'
train_file = '../data/mnist_train.csv'
x_train, y_train = process_csv(train_file)
x_test, y_test = process_csv(test_file)
model = NeuralNetwork()
model.addLayer(InputLayer((1, 784)))
model.addLayer(Dense(neuron_count=300, activation=tanh()))
model.addLayer(Dense(neuron_count=10, activation=sigmoid()))
model.compile(loss=CrossEntropy(), optimizer=SGD(alpha=0.000006))
train_loss, train_acc, val_loss, val_acc = model.fit(x_test,
y_test,
validation_set=True,
validation_split=0.1,
epochs=1,
batch_size=100)