def _gradients(self, thetas, X, y):
# Load vars from neural net
nl = self.neuralnet.num_layers()
# Init variables
DELTA = [None] * (nl)
GRADIENTS = [None] * nl
delta = [None] * (nl + 1)
m = X.shape[0]
# Do forward propagation, but here we don't need the result but the built vars
_, a, z = self._forwardpropagate(thetas, X)
delta[nl] = npe.add_ones(a[nl] - y).T
# Calculate deltas
for i in reversed(range(1, nl)):
delta[i] = np.dot(thetas[i], delta[i+1][1:]) \
* npe.add_ones(NeuralNetClassification.activate(z[i].T, True), 0)
# Calculate DELTAS
for i in range(0, nl):
DELTA[i] = np.dot(delta[i + 1][1:], a[i]).T
# Calculate GRADIENTS
for i in range(0, nl):
GRADIENTS[i] = (1 / m) * DELTA[i]
return GRADIENTS
def train(self,
X,
y,
learning_rate=0.1,
cost_threshold=1e-6,
diff_threshold=1e-16,
max_iter=10000,
min_iter=0,
lambda_=3):
nclass = NeuralNetClassification(self)
#self.thetas = npe.reshape_for_neuralnet(fmin_cg(
# f=nclass._flat_cost,
# x0=npe.flatten(self.thetas),
# fprime=nclass._flat_gradients,
# args=(X, y),
# epsilon=learning_rate,
# # gtol=gtol,
# disp=False,
# maxiter=50), self)
self.thetas = npe.reshape_for_neuralnet(
gradient_descent(start_thetas=npe.flatten(self.thetas),
cost_func=nclass._flat_cost,
gradient_func=nclass._flat_gradients,
args=(X, y, lambda_),
learning_rate=learning_rate,
min_iter=min_iter,
max_iter=max_iter,
cost_threshold=cost_threshold,
diff_threshold=diff_threshold), self)
def _flat_forwardpropagate(self, flat_thetas, X, *args):
thetas = npe.reshape_for_neuralnet(flat_thetas,
self.neuralnet)
r,_,_ = self._forwardpropagate(thetas, self.neuralnet.num_layers(), X)
return npe.flatten(r)
def rand_init(self):
self.thetas = [None] * self.num_layers()
for i in range(0, self.num_layers()):
input = self.input_variables if i == 0 else self.hidden_units
output = self.output_variables if i == self.hidden_layers else self.hidden_units
self.thetas[i] = npe.random_matrix(input,
output,
epsilon=self.epsilon)
def _regularized_gradients(self, thetas, X, y, l):
# Calculate gradients as usual
gradients = self._gradients(thetas, X, y)
# Number of examples & layers
m = X.shape[0]
nl = self.neuralnet.num_layers()
for i in range(0, nl):
gradients[i] += npe.add_zeros((l / m) * thetas[i][:, 1:], 1)
return gradients
def numerical_gradients(self, X, y, epsilon=1e-4, lambda_=3):
flat_thetas = npe.flatten(self.thetas)
numgrad = np.zeros(flat_thetas.shape)
perturb = np.zeros(flat_thetas.shape)
for i in range(0, flat_thetas.size):
perturb[i] = epsilon
loss1 = self.get_algorithms()._flat_cost(flat_thetas - perturb, X,
y, lambda_)
loss2 = self.get_algorithms()._flat_cost(flat_thetas + perturb, X,
y, lambda_)
numgrad[i] = (loss2 - loss1) / (2 * epsilon)
perturb[i] = 0
return numgrad
def _forwardpropagate(self, thetas, X):
# Load vars from neural net
nl = self.neuralnet.num_layers()
# Init variables
a = [None] * (
nl + 1
) # Activation values, equal to z with applied activation method
z = [None] * (nl + 1) # z values
a[0] = X # Starting with X as input
# Iterate over values to calculate a,z
for i in range(0, nl):
a[i] = npe.add_ones(a[i])
z[i + 1] = np.dot(a[i], thetas[i])
a[i + 1] = NeuralNetClassification.activate(
z[i + 1]) # z for next level, i+1
# Return last z values, a's, and z's
return a[nl], a, z
def check_gradients(self,
X,
y,
do_print=False,
imprecision=1e-8,
epsilon=1e-8,
lambda_=3):
flat_thetas = npe.flatten(self.thetas)
algorithmic = self.get_algorithms()._flat_gradients(
flat_thetas, X, y, lambda_)
numerical = self.numerical_gradients(X, y, epsilon, lambda_)
all_match = True
for i in range(0, flat_thetas.size):
is_match = abs(algorithmic[i] - numerical[i]) < imprecision
all_match = all_match and is_match
if (do_print):
print(i, numerical[i], algorithmic[i], is_match)
if (do_print and all_match):
print("All gradients are matching")
elif (do_print and not all_match):
print("Not all gradients are matching")
return all_match
def _flat_regularized_gradients(self, flat_thetas, X, y, l, *args):
thetas = npe.reshape_for_neuralnet(flat_thetas, self.neuralnet)
gradients = self._regularized_gradients(thetas, X, y, l)
f = npe.flatten(gradients)
return f
def _flat_regularized_cost(self, flat_thetas, X, y, l, *args):
thetas = npe.reshape_for_neuralnet(flat_thetas, self.neuralnet)
return self._regularized_cost(thetas, X, y, l)
def _flat_cost(self, flat_thetas, X, y, *args):
thetas = npe.reshape_for_neuralnet(flat_thetas, self.neuralnet)
return self._cost(thetas, X, y)