def forward_pass(self, X, training=True):
# Initialize running mean and variance if first run
if self.running_mean is None:
self.running_mean = R.mean(X, axis=0)
self.running_var = R.variance(X, axis=0)
if training and self.trainable:
mean = R.mean(X, axis=0)
var = R.variance(X, axis=0)
self.running_mean = self.momentum * self.running_mean + (
R.t(1) - self.momentum) * mean
self.running_var = self.momentum * self.running_var + (
R.t(1) - self.momentum) * var
else:
mean = self.running_mean
var = self.running_var
# Statistics saved for backward pass
self.X_centered = X - mean
self.stddev_inv = R.div(R.t(1), R.square_root(var + self.eps))
X_norm = self.X_centered * self.stddev_inv
output = self.gamma * X_norm + self.beta
return output
def test_on_batch(self, X, y):
""" Evaluates the model over a single batch of samples """
y_pred = self._forward_pass(X, training=False)
loss = R.mean(self.loss_function.loss(y, y_pred))
acc = self.loss_function.acc(y, y_pred)
return loss, acc
def train_on_batch(self, X, y):
""" Single gradient update over one batch of samples """
y_pred = self._forward_pass(X)
loss = R.mean(self.loss_function.loss(y, y_pred))
# print(' Loss: ', loss())
acc = self.loss_function.acc(y, y_pred)
# Calculate the gradient of the loss function wrt y_pred
loss_grad = self.loss_function.gradient(y, y_pred)
# print(' Loss Gradient: ', loss_grad())
# Backpropagate. Update weights
self._backward_pass(loss_grad=loss_grad)
return loss, acc