def test_neuralNetwork_adam():
from sklearn.neural_network._stochastic_optimizers import AdamOptimizer
np.random.seed(2019)
X = np.random.normal(size=(1, 500))
target = 3.9285985 * X
nn = NeuralNetwork(inputs=1,
neurons=3,
outputs=1,
activations='sigmoid',
silent=True)
nn.addLayer()
nn.addLayer()
nn.addOutputLayer(activations='identity')
learning_rate = 0.001
yhat = nn.forward_pass(X)
nn.backpropagation(yhat.T, target.T)
nn.learning_rate = learning_rate
nn.initializeAdam()
nn.adam()
skl_adam = AdamOptimizer(params=nn.param, learning_rate_init=learning_rate)
upd = skl_adam._get_updates(nn.grad)
for update_nn, update_skl in zip(nn.change, upd):
assert update_nn == pytest.approx(update_skl)
def test_adam_optimizer():
params = [np.zeros(shape) for shape in shapes]
lr = 0.001
epsilon = 1e-8
for beta_1 in np.arange(0.9, 1.0, 0.05):
for beta_2 in np.arange(0.995, 1.0, 0.001):
optimizer = AdamOptimizer(params, lr, beta_1, beta_2, epsilon)
ms = [np.random.random(shape) for shape in shapes]
vs = [np.random.random(shape) for shape in shapes]
t = 10
optimizer.ms = ms
optimizer.vs = vs
optimizer.t = t - 1
grads = [np.random.random(shape) for shape in shapes]
ms = [beta_1 * m + (1 - beta_1) * grad
for m, grad in zip(ms, grads)]
vs = [beta_2 * v + (1 - beta_2) * (grad ** 2)
for v, grad in zip(vs, grads)]
learning_rate = lr * np.sqrt(1 - beta_2 ** t) / (1 - beta_1**t)
updates = [-learning_rate * m / (np.sqrt(v) + epsilon)
for m, v in zip(ms, vs)]
expected = [param + update
for param, update in zip(params, updates)]
optimizer.update_params(grads)
for exp, param in zip(expected, optimizer.params):
assert_array_equal(exp, param)
class AdamWrapper(UpdadteFunction):
def __init__(self, flags, agents, gpu_nr, memory_fraction, weights):
super().__init__(flags, agents, gpu_nr, memory_fraction, weights)
self.adam = AdamOptimizer(weights,
learning_rate_init=1e-3 / self.agents) #
def __call__(self, weights, update, gradients, staleness, epoch):
"""
:param weights: Copy of the model weights
:param update: Curent update
:param gradients: List of gradients
:param staleness: Staleness of each gradient
:param epoch: Current epoch
"""
grads = np.mean(gradients, axis=0)
self.adam.update_params(grads)
return self.adam.params
def close(self):
pass
def test_adam_optimizer():
params = [np.zeros(shape) for shape in shapes]
lr = 0.001
epsilon = 1e-8
for beta_1 in np.arange(0.9, 1.0, 0.05):
for beta_2 in np.arange(0.995, 1.0, 0.001):
optimizer = AdamOptimizer(params, lr, beta_1, beta_2, epsilon)
ms = [np.random.random(shape) for shape in shapes]
vs = [np.random.random(shape) for shape in shapes]
t = 10
optimizer.ms = ms
optimizer.vs = vs
optimizer.t = t - 1
grads = [np.random.random(shape) for shape in shapes]
ms = [beta_1 * m + (1 - beta_1) * grad
for m, grad in zip(ms, grads)]
vs = [beta_2 * v + (1 - beta_2) * (grad ** 2)
for v, grad in zip(vs, grads)]
learning_rate = lr * np.sqrt(1 - beta_2 ** t) / (1 - beta_1**t)
updates = [-learning_rate * m / (np.sqrt(v) + epsilon)
for m, v in zip(ms, vs)]
expected = [param + update
for param, update in zip(params, updates)]
optimizer.update_params(grads)
for exp, param in zip(expected, optimizer.params):
assert_array_equal(exp, param)
def __init__(self, flags, agents, gpu_nr, memory_fraction, weights):
super().__init__(flags, agents, gpu_nr, memory_fraction, weights)
self.adam = AdamOptimizer(weights,
learning_rate_init=1e-3 / self.agents) #
def _fit_stochastic(self, X, y, X_val, y_val, activations, deltas,
coef_grads, intercept_grads, layer_units, incremental):
if not incremental or not hasattr(self, '_optimizer'):
params = self.coefs_ + self.intercepts_
if self.solver == 'sgd':
self._optimizer = SGDOptimizer(params, self.learning_rate_init,
self.learning_rate,
self.momentum,
self.nesterovs_momentum,
self.power_t)
elif self.solver == 'adam':
self._optimizer = AdamOptimizer(params,
self.learning_rate_init,
self.beta_1, self.beta_2,
self.epsilon)
# early_stopping in partial_fit doesn't make sense
early_stopping = self.early_stopping and not incremental
n_samples = X.shape[0]
if self.batch_size == 'auto':
batch_size = min(200, n_samples)
else:
batch_size = np.clip(self.batch_size, 1, n_samples)
try:
for it in range(self.max_iter):
X, y = shuffle(X, y, random_state=self._random_state)
accumulated_loss = 0.0
for batch_slice in gen_batches(n_samples, batch_size):
activations[0] = X[batch_slice]
batch_loss, coef_grads, intercept_grads = self._backprop(
X[batch_slice], y[batch_slice], activations, deltas,
coef_grads, intercept_grads)
accumulated_loss += batch_loss * (batch_slice.stop -
batch_slice.start)
# update weights
grads = coef_grads + intercept_grads
self._optimizer.update_params(grads)
# Get val path
activations[0] = X_val
activations = self._forward_pass(activations)
loss_val = LOSS_FUNCTIONS[self.loss](y_val, activations[-1])
self.lpath['val'].append(loss_val)
self.n_iter_ += 1
self.loss_ = accumulated_loss / X.shape[0]
self.t_ += n_samples
self.loss_curve_.append(self.loss_)
self.lpath['train'].append(self.loss_)
if self.verbose:
print("Iteration %d, loss = %.8f" %
(self.n_iter_, self.loss_))
# update no_improvement_count based on training loss or
# validation score according to early_stopping
self._update_no_improvement_count(early_stopping, X_val, y_val)
# for learning rate that needs to be updated at iteration end
self._optimizer.iteration_ends(self.t_)
if self._no_improvement_count > self.n_iter_no_change:
# not better than last `n_iter_no_change` iterations by tol
# stop or decrease learning rate
if early_stopping:
msg = ("Validation score did not improve more than "
"tol=%f for %d consecutive epochs." %
(self.tol, self.n_iter_no_change))
else:
msg = ("Training loss did not improve more than tol=%f"
" for %d consecutive epochs." %
(self.tol, self.n_iter_no_change))
is_stopping = self._optimizer.trigger_stopping(
msg, self.verbose)
if is_stopping:
break
else:
self._no_improvement_count = 0
if incremental:
break
if self.n_iter_ == self.max_iter:
warnings.warn(
"Stochastic Optimizer: Maximum iterations (%d) "
"reached and the optimization hasn't converged yet." %
self.max_iter, ConvergenceWarning)
except KeyboardInterrupt:
warnings.warn("Training interrupted by user.")
if early_stopping:
# restore best weights
self.coefs_ = self._best_coefs
self.intercepts_ = self._best_intercepts
class MLPRegressor(BaseMLP):
def __init__(self,
bound=(-10, 10),
popsize=5,
strategy='best1bin',
recombination=None,
T=0.01,
stepsize=0.01,
*args,
**kwargs):
super(MLPRegressor, self).__init__(*args, **kwargs)
self.lpath = dict(train=[], val=[])
self.n_iter_no_change = kwargs.get('n_iter_no_change', 1)
self._iter = 0
# parameter for differents evolution
self.bound = bound
self.popsize = popsize
self.straegy = strategy
self.recombination = recombination
# parameter for anneal
self.T = T
self.stepsize = stepsize
def _calculator_activations(self, packed_parameters):
"""Extract the coefficients and intercepts from packed_parameters."""
coefs_ = []
intercepts_ = []
for i in range(self.n_layers_ - 1):
start, end, shape = self._coef_indptr[i]
coefs_.append(np.reshape(packed_parameters[start:end], shape))
start, end = self._intercept_indptr[i]
intercepts_.append(packed_parameters[start:end])
"""Perform a forward pass on the network by computing the values
of the neurons in the hidden layers and the output layer.
Parameters
----------
activations : list, length = n_layers - 1
The ith element of the list holds the values of the ith layer.
with_output_activation : bool, default True
If True, the output passes through the output activation
function, which is either the softmax function or the
logistic function
"""
# calculator predict validation set
activations = list(self.val_activations)
hidden_activation = ACTIVATIONS[self.activation]
# Iterate over the hidden layers
for i in range(self.n_layers_ - 1):
activations[i + 1] = safe_sparse_dot(activations[i], coefs_[i])
activations[i + 1] += intercepts_[i]
# For the hidden layers
if (i + 1) != (self.n_layers_ - 1):
activations[i + 1] = hidden_activation(activations[i + 1])
# For the last layer
output_activation = ACTIVATIONS[self.out_activation_]
activations[i + 1] = output_activation(activations[i + 1])
p_val = list(activations[-1])
# calculator predict trainset
activations = list(self.train_activations)
hidden_activation = ACTIVATIONS[self.activation]
# Iterate over the hidden layers
for i in range(self.n_layers_ - 1):
activations[i + 1] = safe_sparse_dot(activations[i], coefs_[i])
activations[i + 1] += intercepts_[i]
# For the hidden layers
if (i + 1) != (self.n_layers_ - 1):
activations[i + 1] = hidden_activation(activations[i + 1])
# For the last layer
output_activation = ACTIVATIONS[self.out_activation_]
activations[i + 1] = output_activation(activations[i + 1])
p_train = list(activations[-1])
return p_train, p_val
def _callback(self, packed_parameters, *args, **kwargs):
p_train, p_val = self._calculator_activations(packed_parameters)
loss_train = LOSS_FUNCTIONS[self.loss](self.y_train, p_train)
loss_val = LOSS_FUNCTIONS[self.loss](self.y_val, p_val)
if self.verbose:
print('Iter %3d loss train %.5f loss val %.5f' % \
(self._iter, loss_train, loss_val))
self._iter += 1
self.lpath['train'].append(loss_train)
self.lpath['val'].append(loss_val)
def _fit_bfgs(self, X, y, X_val, Y_val, activations, deltas, coef_grads,
intercept_grads, layer_units):
# Store meta information for the parameters
self._coef_indptr = []
self._intercept_indptr = []
start = 0
# Save sizes and indices of coefficients for faster unpacking
for i in range(self.n_layers_ - 1):
n_fan_in, n_fan_out = layer_units[i], layer_units[i + 1]
end = start + (n_fan_in * n_fan_out)
self._coef_indptr.append((start, end, (n_fan_in, n_fan_out)))
start = end
# Save sizes and indices of intercepts for faster unpacking
for i in range(self.n_layers_ - 1):
end = start + layer_units[i + 1]
self._intercept_indptr.append((start, end))
start = end
# Run BFGS
packed_coef_inter = _pack(self.coefs_, self.intercepts_)
if self.verbose is True or self.verbose >= 1:
iprint = 1
else:
iprint = -1
optimal_parameters, self.loss_, d, Bopt, func_calls, grad_calls, warnflag = \
optimize.fmin_bfgs(x0=packed_coef_inter,
f=self._loss_func,
fprime=self._grad_func,
maxiter=self.max_iter,
disp=False,
gtol=self.tol,
args=(X, y, activations, deltas, coef_grads, intercept_grads),
full_output=True,
callback=self._callback)
self._unpack(optimal_parameters)
def _fit_lbfgs(self, X, y, X_val, Y_val, activations, deltas, coef_grads,
intercept_grads, layer_units):
# Store meta information for the parameters
self._coef_indptr = []
self._intercept_indptr = []
start = 0
# Save sizes and indices of coefficients for faster unpacking
for i in range(self.n_layers_ - 1):
n_fan_in, n_fan_out = layer_units[i], layer_units[i + 1]
end = start + (n_fan_in * n_fan_out)
self._coef_indptr.append((start, end, (n_fan_in, n_fan_out)))
start = end
# Save sizes and indices of intercepts for faster unpacking
for i in range(self.n_layers_ - 1):
end = start + layer_units[i + 1]
self._intercept_indptr.append((start, end))
start = end
# Run LBFGS
packed_coef_inter = _pack(self.coefs_, self.intercepts_)
if self.verbose is True or self.verbose >= 1:
iprint = 1
else:
iprint = -1
optimal_parameters, self.loss_, d = optimize.fmin_l_bfgs_b(
x0=packed_coef_inter,
func=self._loss_grad_lbfgs,
maxfun=self.max_iter,
iprint=-1,
pgtol=self.tol,
args=(X, y, activations, deltas, coef_grads, intercept_grads),
callback=self._callback)
self._unpack(optimal_parameters)
def _fit_evol(self, X, y, X_val, Y_val, activations, deltas, coef_grads,
intercept_grads, layer_units):
# Store meta information for the parameters
self._coef_indptr = []
self._intercept_indptr = []
start = 0
# Save sizes and indices of coefficients for faster unpacking
for i in range(self.n_layers_ - 1):
n_fan_in, n_fan_out = layer_units[i], layer_units[i + 1]
end = start + (n_fan_in * n_fan_out)
self._coef_indptr.append((start, end, (n_fan_in, n_fan_out)))
start = end
# Save sizes and indices of intercepts for faster unpacking
for i in range(self.n_layers_ - 1):
end = start + layer_units[i + 1]
self._intercept_indptr.append((start, end))
start = end
# Run evolution
packed_coef_inter = _pack(self.coefs_, self.intercepts_)
if self.verbose is True or self.verbose >= 1:
iprint = 1
else:
iprint = -1
bounds = list()
for i in range(len(packed_coef_inter)):
bounds.append(self.bound)
result = optimize.differential_evolution(func=self._loss_func,
bounds=bounds,
maxiter=self.max_iter,
disp=False,
polish=True,
init='latinhypercube',
popsize=self.popsize,
strategy=self.straegy,
seed=self.random_state,
args=(X, y, activations,
deltas, coef_grads,
intercept_grads),
callback=self._callback)
optimal_parameters = result.x
self.loss_ = result.fun
self._unpack(optimal_parameters)
def _fit_cg(self, X, y, X_val, Y_val, activations, deltas, coef_grads,
intercept_grads, layer_units):
# Store meta information for the parameters
self._coef_indptr = []
self._intercept_indptr = []
start = 0
# Save sizes and indices of coefficients for faster unpacking
for i in range(self.n_layers_ - 1):
n_fan_in, n_fan_out = layer_units[i], layer_units[i + 1]
end = start + (n_fan_in * n_fan_out)
self._coef_indptr.append((start, end, (n_fan_in, n_fan_out)))
start = end
# Save sizes and indices of intercepts for faster unpacking
for i in range(self.n_layers_ - 1):
end = start + layer_units[i + 1]
self._intercept_indptr.append((start, end))
start = end
# Run CG
packed_coef_inter = _pack(self.coefs_, self.intercepts_)
optimal_parameters, self.loss_, func_calls, grad_calls, d = \
optimize.fmin_cg(x0=packed_coef_inter,
f=self._loss_func,
fprime=self._grad_func,
maxiter=self.max_iter,
disp=False,
epsilon=self.epsilon,
gtol=self.tol,
args=(X, y, activations, deltas, coef_grads, intercept_grads),
callback=self._callback,
full_output=True)
self._unpack(optimal_parameters)
def _fit_ncg(self, X, y, X_val, Y_val, activations, deltas, coef_grads,
intercept_grads, layer_units):
# Store meta information for the parameters
self._coef_indptr = []
self._intercept_indptr = []
start = 0
# Save sizes and indices of coefficients for faster unpacking
for i in range(self.n_layers_ - 1):
n_fan_in, n_fan_out = layer_units[i], layer_units[i + 1]
end = start + (n_fan_in * n_fan_out)
self._coef_indptr.append((start, end, (n_fan_in, n_fan_out)))
start = end
# Save sizes and indices of intercepts for faster unpacking
for i in range(self.n_layers_ - 1):
end = start + layer_units[i + 1]
self._intercept_indptr.append((start, end))
start = end
# Run Newton-CG
packed_coef_inter = _pack(self.coefs_, self.intercepts_)
optimal_parameters, self.loss_, func_calls, grad_calls, h_calls, d = \
optimize.fmin_ncg(x0=packed_coef_inter,
f=self._loss_func,
fprime=self._grad_func,
maxiter=200,
# maxiter=self.max_iter,
disp=True,
args=(X, y, activations, deltas, coef_grads, intercept_grads),
callback=self._callback,
full_output=True)
self._unpack(optimal_parameters)
def _fit_basinhopping(self, X, y, X_val, Y_val, activations, deltas,
coef_grads, intercept_grads, layer_units):
# Store meta information for the parameters
self._coef_indptr = []
self._intercept_indptr = []
start = 0
# Save sizes and indices of coefficients for faster unpacking
for i in range(self.n_layers_ - 1):
n_fan_in, n_fan_out = layer_units[i], layer_units[i + 1]
end = start + (n_fan_in * n_fan_out)
self._coef_indptr.append((start, end, (n_fan_in, n_fan_out)))
start = end
# Save sizes and indices of intercepts for faster unpacking
for i in range(self.n_layers_ - 1):
end = start + layer_units[i + 1]
self._intercept_indptr.append((start, end))
start = end
# Run Basinhopping
packed_coef_inter = _pack(self.coefs_, self.intercepts_)
minimizer_kwargs = {
'method': 'L-BFGS-B',
'args': (X, y, activations, deltas, coef_grads, intercept_grads)
}
result = optimize.basinhopping(x0=packed_coef_inter,
T=self.T,
stepsize=self.stepsize,
func=self._loss_func,
niter=self.max_iter,
callback=self._callback,
minimizer_kwargs=minimizer_kwargs)
optimal_parameters = result.x
self.loss = result.fun
self._unpack(optimal_parameters)
def _fit_anneal(self, X, y, X_val, Y_val, activations, deltas, coef_grads,
intercept_grads, layer_units):
if scipy.__version__ == '0.14.0':
# Store meta information for the parameters
self._coef_indptr = []
self._intercept_indptr = []
start = 0
# Save sizes and indices of coefficients for faster unpacking
for i in range(self.n_layers_ - 1):
n_fan_in, n_fan_out = layer_units[i], layer_units[i + 1]
end = start + (n_fan_in * n_fan_out)
self._coef_indptr.append((start, end, (n_fan_in, n_fan_out)))
start = end
# Save sizes and indices of intercepts for faster unpacking
for i in range(self.n_layers_ - 1):
end = start + layer_units[i + 1]
self._intercept_indptr.append((start, end))
start = end
# Run Simulated Annealing
packed_coef_inter = _pack(self.coefs_, self.intercepts_)
result = optimize.anneal(x0=packed_coef_inter,
T0=self.T,
stepsize=self.stepsize,
func=self._loss_func,
maxiter=self.max_iter,
args=(X, y, activations, deltas,
coef_grads, intercept_grads))
optimal_parameters = result.x
self.loss = result.fun
self._unpack(optimal_parameters)
else:
raise ImportError('Anneal method require scipy version <= 0.14.0')
def _grad_func(self, packed_coef_inter, X, y, activations, deltas,
coef_grads, intercept_grads):
loss, coef_grads, intercept_grads = \
self._backprop(X, y, activations, deltas,
coef_grads, intercept_grads)
grad = _pack(coef_grads, intercept_grads)
return grad
def _loss_func(self, packed_coef_inter, X, y, activations, deltas,
coef_grads, intercept_grads):
"""Compute the MLP loss function and its corresponding derivatives
with respect to the different parameters given in the initialization.
Returned loss are packed in a single vector so it can be used
in cg
Parameters
----------
packed_coef_inter : array-like
A vector comprising the flattened coefficients and intercepts.
X : {array-like, sparse matrix}, shape (n_samples, n_features)
The input data.
y : array-like, shape (n_samples,)
The target values.
activations : list, length = n_layers - 1
The ith element of the list holds the values of the ith layer.
deltas : list, length = n_layers - 1
The ith element of the list holds the difference between the
activations of the i + 1 layer and the backpropagated error.
More specifically, deltas are gradients of loss with respect to z
in each layer, where z = wx + b is the value of a particular layer
before passing through the activation function
coef_grads : list, length = n_layers - 1
The ith element contains the amount of change used to update the
coefficient parameters of the ith layer in an iteration.
intercept_grads : list, length = n_layers - 1
The ith element contains the amount of change used to update the
intercept parameters of the ith layer in an iteration.
Returns
-------
loss : float
"""
self._unpack(packed_coef_inter)
loss, coef_grads, intercept_grads = self._backprop(
X, y, activations, deltas, coef_grads, intercept_grads)
self.n_iter_ += 1
return loss
def _fit_stochastic(self, X, y, X_val, y_val, activations, deltas,
coef_grads, intercept_grads, layer_units, incremental):
if not incremental or not hasattr(self, '_optimizer'):
params = self.coefs_ + self.intercepts_
if self.solver == 'sgd':
self._optimizer = SGDOptimizer(params, self.learning_rate_init,
self.learning_rate,
self.momentum,
self.nesterovs_momentum,
self.power_t)
elif self.solver == 'adam':
self._optimizer = AdamOptimizer(params,
self.learning_rate_init,
self.beta_1, self.beta_2,
self.epsilon)
# early_stopping in partial_fit doesn't make sense
early_stopping = self.early_stopping and not incremental
n_samples = X.shape[0]
if self.batch_size == 'auto':
batch_size = min(200, n_samples)
else:
batch_size = np.clip(self.batch_size, 1, n_samples)
try:
for it in range(self.max_iter):
X, y = shuffle(X, y, random_state=self._random_state)
accumulated_loss = 0.0
for batch_slice in gen_batches(n_samples, batch_size):
activations[0] = X[batch_slice]
batch_loss, coef_grads, intercept_grads = self._backprop(
X[batch_slice], y[batch_slice], activations, deltas,
coef_grads, intercept_grads)
accumulated_loss += batch_loss * (batch_slice.stop -
batch_slice.start)
# update weights
grads = coef_grads + intercept_grads
self._optimizer.update_params(grads)
# Get val path
activations[0] = X_val
activations = self._forward_pass(activations)
loss_val = LOSS_FUNCTIONS[self.loss](y_val, activations[-1])
self.lpath['val'].append(loss_val)
self.n_iter_ += 1
self.loss_ = accumulated_loss / X.shape[0]
self.t_ += n_samples
self.loss_curve_.append(self.loss_)
self.lpath['train'].append(self.loss_)
if self.verbose:
print("Iteration %d, loss = %.8f" %
(self.n_iter_, self.loss_))
# update no_improvement_count based on training loss or
# validation score according to early_stopping
self._update_no_improvement_count(early_stopping, X_val, y_val)
# for learning rate that needs to be updated at iteration end
self._optimizer.iteration_ends(self.t_)
if self._no_improvement_count > self.n_iter_no_change:
# not better than last `n_iter_no_change` iterations by tol
# stop or decrease learning rate
if early_stopping:
msg = ("Validation score did not improve more than "
"tol=%f for %d consecutive epochs." %
(self.tol, self.n_iter_no_change))
else:
msg = ("Training loss did not improve more than tol=%f"
" for %d consecutive epochs." %
(self.tol, self.n_iter_no_change))
is_stopping = self._optimizer.trigger_stopping(
msg, self.verbose)
if is_stopping:
break
else:
self._no_improvement_count = 0
if incremental:
break
if self.n_iter_ == self.max_iter:
warnings.warn(
"Stochastic Optimizer: Maximum iterations (%d) "
"reached and the optimization hasn't converged yet." %
self.max_iter, ConvergenceWarning)
except KeyboardInterrupt:
warnings.warn("Training interrupted by user.")
if early_stopping:
# restore best weights
self.coefs_ = self._best_coefs
self.intercepts_ = self._best_intercepts
def _backprop(self, X, y, activations, deltas, coef_grads,
intercept_grads):
"""Compute the MLP loss function and its corresponding derivatives
with respect to each parameter: weights and bias vectors.
Parameters
----------
X : {array-like, sparse matrix}, shape (n_samples, n_features)
The input data.
y : array-like, shape (n_samples,)
The target values.
activations : list, length = n_layers - 1
The ith element of the list holds the values of the ith layer.
deltas : list, length = n_layers - 1
The ith element of the list holds the difference between the
activations of the i + 1 layer and the backpropagated error.
More specifically, deltas are gradients of loss with respect to z
in each layer, where z = wx + b is the value of a particular layer
before passing through the activation function
coef_grads : list, length = n_layers - 1
The ith element contains the amount of change used to update the
coefficient parameters of the ith layer in an iteration.
intercept_grads : list, length = n_layers - 1
The ith element contains the amount of change used to update the
intercept parameters of the ith layer in an iteration.
flag: True if calculator train error
Returns
-------
loss : float
coef_grads : list, length = n_layers - 1
intercept_grads : list, length = n_layers - 1
"""
n_samples = X.shape[0]
# Forward propagate
activations = self._forward_pass(activations)
# Get loss
loss_func_name = self.loss
if loss_func_name == 'log_loss' and self.out_activation_ == 'logistic':
loss_func_name = 'binary_log_loss'
loss = LOSS_FUNCTIONS[loss_func_name](y, activations[-1])
# Add L2 regularization term to loss
values = np.sum(
np.array([np.dot(s.ravel(), s.ravel()) for s in self.coefs_]))
loss += (0.5 * self.alpha) * values / n_samples
# Backward propagate
last = self.n_layers_ - 2
# The calculation of delta[last] here works with following
# combinations of output activation and loss function:
# sigmoid and binary cross entropy, softmax and categorical cross
# entropy, and identity with squared loss
deltas[last] = activations[-1] - y
# Compute gradient for the last layer
coef_grads, intercept_grads = self._compute_loss_grad(
last, n_samples, activations, deltas, coef_grads, intercept_grads)
# Iterate over the hidden layers
for i in range(self.n_layers_ - 2, 0, -1):
deltas[i - 1] = safe_sparse_dot(deltas[i], self.coefs_[i].T)
inplace_derivative = DERIVATIVES[self.activation]
inplace_derivative(activations[i], deltas[i - 1])
coef_grads, intercept_grads = self._compute_loss_grad(
i - 1, n_samples, activations, deltas, coef_grads,
intercept_grads)
return loss, coef_grads, intercept_grads
def _fit(self, X, y, incremental=False):
X, X_val, y, Y_val = train_test_split(X, y, test_size=0.1)
# Make sure self.hidden_layer_sizes is a list
hidden_layer_sizes = self.hidden_layer_sizes
if not hasattr(hidden_layer_sizes, "__iter__"):
hidden_layer_sizes = [hidden_layer_sizes]
hidden_layer_sizes = list(hidden_layer_sizes)
# Validate input parameters.
self._validate_hyperparameters()
if np.any(np.array(hidden_layer_sizes) <= 0):
raise ValueError("hidden_layer_sizes must be > 0, got %s." %
hidden_layer_sizes)
X, y = self._validate_input(X, y, incremental)
n_samples, n_features = X.shape
# Ensure y is 2D
if y.ndim == 1:
y = y.reshape((-1, 1))
self.n_outputs_ = y.shape[1]
layer_units = ([n_features] + hidden_layer_sizes + [self.n_outputs_])
# check random state
self._random_state = check_random_state(self.random_state)
if not hasattr(self, 'coefs_') or (not self.warm_start
and not incremental):
# First time training the model
self._initialize(y, layer_units)
# lbfgs does not support mini-batches
if self.solver == 'lbfgs' or self.solver == 'cg':
batch_size = n_samples
elif self.batch_size == 'auto':
batch_size = min(200, n_samples)
else:
if self.batch_size < 1 or self.batch_size > n_samples:
warnings.warn("Got `batch_size` less than 1 or larger than "
"sample size. It is going to be clipped")
batch_size = np.clip(self.batch_size, 1, n_samples)
# Initialize lists
activations = [X]
activations.extend(
np.empty((batch_size, n_fan_out)) for n_fan_out in layer_units[1:])
self.val_activations = list(activations)
self.train_activations = list(activations)
self.val_activations[0] = [X_val]
self.train_activations[0] = [X]
self.y_val = Y_val
self.y_train = y
deltas = [np.empty_like(a_layer) for a_layer in activations]
coef_grads = [
np.empty((n_fan_in_, n_fan_out_))
for n_fan_in_, n_fan_out_ in zip(layer_units[:-1], layer_units[1:])
]
intercept_grads = [
np.empty(n_fan_out_) for n_fan_out_ in layer_units[1:]
]
# Run the Stochastic optimization solver
if self.solver in _STOCHASTIC_SOLVERS:
self._fit_stochastic(X, y, X_val, Y_val, activations, deltas,
coef_grads, intercept_grads, layer_units,
incremental)
# Run the LBFGS solver
elif self.solver == 'lbfgs':
self._fit_lbfgs(X, y, X_val, Y_val, activations, deltas,
coef_grads, intercept_grads, layer_units)
# Run the BFGS solver
elif self.solver == 'bfgs':
self._fit_bfgs(X, y, X_val, Y_val, activations, deltas, coef_grads,
intercept_grads, layer_units)
# Run the Conjugate Gradient
elif self.solver == 'cg':
self._fit_cg(X, y, X_val, Y_val, activations, deltas, coef_grads,
intercept_grads, layer_units)
# Run the Newton-CG
elif self.solver == 'ncg':
self._fit_ncg(X, y, X_val, Y_val, activations, deltas, coef_grads,
intercept_grads, layer_units)
# Run the Evolution
elif self.solver == 'evol':
self._fit_evol(X, y, X_val, Y_val, activations, deltas, coef_grads,
intercept_grads, layer_units)
# Run the Basinhopping
elif self.solver == 'basinhopping':
self._fit_basinhopping(X, y, X_val, Y_val, activations, deltas,
coef_grads, intercept_grads, layer_units)
# Run the Simulated Annealing
elif self.solver == 'anneal':
self._fit_anneal(X, y, X_val, Y_val, activations, deltas,
coef_grads, intercept_grads, layer_units)
# Delete dataset is member class
del self.train_activations
del self.val_activations
del self.y_train
del self.y_val
return self
def _validate_hyperparameters(self):
if not isinstance(self.shuffle, bool):
raise ValueError("shuffle must be either True or False, got %s." %
self.shuffle)
if self.max_iter <= 0:
raise ValueError("max_iter must be > 0, got %s." % self.max_iter)
if self.alpha < 0.0:
raise ValueError("alpha must be >= 0, got %s." % self.alpha)
if (self.learning_rate in ["constant", "invscaling", "adaptive"]
and self.learning_rate_init <= 0.0):
raise ValueError("learning_rate_init must be > 0, got %s." %
self.learning_rate)
if self.momentum > 1 or self.momentum < 0:
raise ValueError("momentum must be >= 0 and <= 1, got %s" %
self.momentum)
if not isinstance(self.nesterovs_momentum, bool):
raise ValueError("nesterovs_momentum must be either True or False,"
" got %s." % self.nesterovs_momentum)
if not isinstance(self.early_stopping, bool):
raise ValueError("early_stopping must be either True or False,"
" got %s." % self.early_stopping)
if self.validation_fraction < 0 or self.validation_fraction >= 1:
raise ValueError("validation_fraction must be >= 0 and < 1, "
"got %s" % self.validation_fraction)
if self.beta_1 < 0 or self.beta_1 >= 1:
raise ValueError("beta_1 must be >= 0 and < 1, got %s" %
self.beta_1)
if self.beta_2 < 0 or self.beta_2 >= 1:
raise ValueError("beta_2 must be >= 0 and < 1, got %s" %
self.beta_2)
if self.epsilon <= 0.0:
raise ValueError("epsilon must be > 0, got %s." % self.epsilon)
# raise ValueError if not registered
supported_activations = ('identity', 'logistic', 'tanh', 'relu')
if self.activation not in supported_activations:
raise ValueError("The activation '%s' is not supported. Supported "
"activations are %s." %
(self.activation, supported_activations))
if self.learning_rate not in ["constant", "invscaling", "adaptive"]:
raise ValueError("learning rate %s is not supported. " %
self.learning_rate)
supported_solvers = _STOCHASTIC_SOLVERS + [
"lbfgs", "bfgs", "cg", "ncg", "evol", "anneal", "basinhopping"
]
if self.solver not in supported_solvers:
raise ValueError("The solver %s is not supported. "
" Expected one of: %s" %
(self.solver, ", ".join(supported_solvers)))
supported_strategy = [
'best1bin', 'best1exp', 'rand1exp', 'randtobest1exp', 'best2exp',
'rand2exp', 'randtobest1bin', 'best2bin', 'rand2bin', 'rand1bin'
]
if self.straegy not in supported_strategy:
raise ValueError("The strategy %s is not supported. "
" Expected one of: %s" %
(self.solver, ", ".join(supported_solvers)))
class ModMLPClassifier(MLPClassifier):
"""
Extension of MLPClassifier class in scikit-learn.
This extension supports the new parameters train_folds and max_fail.
param train_folds: number of folds merged in training set.
If train_folds is None the default train_test_split is used to obtain the validation set based on validation
fraction parameter, if is a number that indicates the number of folds of the training set: the last fold is selected
as validation fold.
param max_fail: overrides the default constant value (2) of max fail in the scikit-learn implementation
"""
def __init__(self,
hidden_layer_sizes=(100, ),
activation="relu",
solver='adam',
alpha=0.0001,
batch_size='auto',
learning_rate="constant",
learning_rate_init=0.001,
power_t=0.5,
max_iter=200,
shuffle=True,
random_state=None,
tol=1e-4,
verbose=False,
warm_start=False,
momentum=0.9,
nesterovs_momentum=True,
early_stopping=False,
validation_fraction=0.1,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-8,
train_folds=None,
max_fail=5):
super().__init__(hidden_layer_sizes, activation, solver, alpha,
batch_size, learning_rate, learning_rate_init,
power_t, max_iter, shuffle, random_state, tol,
verbose, warm_start, momentum, nesterovs_momentum,
early_stopping, validation_fraction, beta_1, beta_2,
epsilon)
# EXTENSION added properties
self.train_folds = train_folds
self.max_fail = max_fail
def _fit_stochastic(self, X, y, activations, deltas, coef_grads,
intercept_grads, layer_units, incremental):
if not incremental or not hasattr(self, '_optimizer'):
params = self.coefs_ + self.intercepts_
if self.solver == 'sgd':
self._optimizer = SGDOptimizer(params, self.learning_rate_init,
self.learning_rate,
self.momentum,
self.nesterovs_momentum,
self.power_t)
elif self.solver == 'adam':
self._optimizer = AdamOptimizer(params,
self.learning_rate_init,
self.beta_1, self.beta_2,
self.epsilon)
# early_stopping in partial_fit doesn't make sense
early_stopping = self.early_stopping and not incremental
if early_stopping:
# EXTENSION train_folds (modifications here)
X, X_val, y, y_val = self._split_train_validation(X, y)
if is_classifier(self):
y_val = self._label_binarizer.inverse_transform(y_val)
else:
X_val = None
y_val = None
n_samples = X.shape[0]
if self.batch_size == 'auto':
batch_size = min(200, n_samples)
else:
batch_size = np.clip(self.batch_size, 1, n_samples)
try:
for it in range(self.max_iter):
X, y = shuffle(X, y, random_state=self._random_state)
accumulated_loss = 0.0
for batch_slice in gen_batches(n_samples, batch_size):
activations[0] = X[batch_slice]
batch_loss, coef_grads, intercept_grads = self._backprop(
X[batch_slice], y[batch_slice], activations, deltas,
coef_grads, intercept_grads)
accumulated_loss += batch_loss * (batch_slice.stop -
batch_slice.start)
# update weights
grads = coef_grads + intercept_grads
self._optimizer.update_params(grads)
self.n_iter_ += 1
self.loss_ = accumulated_loss / X.shape[0]
self.t_ += n_samples
self.loss_curve_.append(self.loss_)
if self.verbose:
print("Iteration %d, loss = %.8f" %
(self.n_iter_, self.loss_))
# update no_improvement_count based on training loss or
# validation score according to early_stopping
self._update_no_improvement_count(early_stopping, X_val, y_val)
# for learning rate that needs to be updated at iteration end
self._optimizer.iteration_ends(self.t_)
# EXTENSION max_fail (modified next line)
if self._no_improvement_count > self.max_fail:
# not better than last two iterations by tol.
# stop or decrease learning rate
if early_stopping:
msg = ("Validation score did not improve more than "
"tol=%f for two consecutive epochs." % self.tol)
else:
msg = ("Training loss did not improve more than tol=%f"
" for two consecutive epochs." % self.tol)
is_stopping = self._optimizer.trigger_stopping(
msg, self.verbose)
if is_stopping:
break
else:
self._no_improvement_count = 0
if incremental:
break
if self.n_iter_ == self.max_iter:
warnings.warn(
"Stochastic Optimizer: Maximum iterations (%d) "
"reached and the optimization hasn't converged yet." %
self.max_iter, ConvergenceWarning)
except KeyboardInterrupt:
warnings.warn("Training interrupted by user.")
if early_stopping:
# restore best weights
self.coefs_ = self._best_coefs
self.intercepts_ = self._best_intercepts
# EXTENSION train_folds (new function)
def _split_train_validation(self, X, y):
if self.train_folds is None or self.train_folds == 0:
X_train, X_test, y_train, y_test = train_test_split(
X,
y,
random_state=self._random_state,
test_size=self.validation_fraction)
else:
split_index = int(
len(y) * ((self.train_folds - 1) / self.train_folds))
X_train = X[0:split_index]
X_test = X[split_index:X.shape[0]]
y_train = y[0:split_index]
y_test = y[split_index:X.shape[0]]
return X_train, X_test, y_train, y_test
def _fit_stochastic(self, X, y, activations, deltas, coef_grads,
intercept_grads, layer_units, incremental):
if not incremental or not hasattr(self, '_optimizer'):
params = self.coefs_ + self.intercepts_
if self.solver == 'sgd':
self._optimizer = SGDOptimizer(
params, self.learning_rate_init, self.learning_rate,
self.momentum, self.nesterovs_momentum, self.power_t)
elif self.solver == 'adam':
self._optimizer = AdamOptimizer(
params, self.learning_rate_init, self.beta_1, self.beta_2,
self.epsilon)
# early_stopping in partial_fit doesn't make sense
early_stopping = self.early_stopping and not incremental
if early_stopping:
X, X_val, y, y_val = train_test_split(
X, y, random_state=self._random_state,
test_size=self.validation_fraction)
if isinstance(self, ClassifierMixin):
y_val = self._label_binarizer.inverse_transform(y_val)
else:
X_val = None
y_val = None
n_samples = X.shape[0]
if self.batch_size == 'auto':
batch_size = min(200, n_samples)
else:
batch_size = np.clip(self.batch_size, 1, n_samples)
'''for explanation start'''
activations_over_all_itr = []
'''for explanation end'''
try:
for it in range(self.max_iter):
X, y = shuffle(X, y, random_state=self._random_state)
accumulated_loss = 0.0
for batch_slice in gen_batches(n_samples, batch_size):
activations[0] = X[batch_slice]
batch_loss, coef_grads, intercept_grads = self._backprop(
X[batch_slice], y[batch_slice], activations, deltas,
coef_grads, intercept_grads)
accumulated_loss += batch_loss * (batch_slice.stop -
batch_slice.start)
# update weights
grads = coef_grads + intercept_grads
self._optimizer.update_params(grads)
'''for explanation start'''
activations_over_all_itr.append(activations)
'''for explanation end'''
self.n_iter_ += 1
self.loss_ = accumulated_loss / X.shape[0]
self.t_ += n_samples
self.loss_curve_.append(self.loss_)
if self.verbose:
print("Iteration %d, loss = %.8f" % (self.n_iter_,
self.loss_))
# update no_improvement_count based on training loss or
# validation score according to early_stopping
self._update_no_improvement_count(early_stopping, X_val, y_val)
# for learning rate that needs to be updated at iteration end
self._optimizer.iteration_ends(self.t_)
if self._no_improvement_count > 2:
# not better than last two iterations by tol.
# stop or decrease learning rate
if early_stopping:
msg = ("Validation score did not improve more than "
"tol=%f for two consecutive epochs." % self.tol)
else:
msg = ("Training loss did not improve more than tol=%f"
" for two consecutive epochs." % self.tol)
is_stopping = self._optimizer.trigger_stopping(
msg, self.verbose)
if is_stopping:
break
else:
self._no_improvement_count = 0
if incremental:
break
if self.n_iter_ == self.max_iter:
warnings.warn(
"Stochastic Optimizer: Maximum iterations (%d) "
"reached and the optimization hasn't converged yet."
% self.max_iter, ConvergenceWarning)
except KeyboardInterrupt:
warnings.warn("Training interrupted by user.")
if early_stopping:
# restore best weights
self.coefs_ = self._best_coefs
self.intercepts_ = self._best_intercepts
'''for explanation start'''
return activations_over_all_itr
'''for explanation end'''
class BaseMultilayerPerceptron_Custom(six.with_metaclass(ABCMeta, BaseEstimator)):
"""Base class for MLP classification and regression.
Warning: This class should not be used directly.
Use derived classes instead.
.. versionadded:: 0.18
"""
@abstractmethod
def __init__(self, hidden_layer_sizes, activation, solver,
alpha, batch_size, learning_rate, learning_rate_init, power_t,
max_iter, loss, shuffle, random_state, tol, verbose,
warm_start, momentum, nesterovs_momentum, early_stopping,
validation_fraction, beta_1, beta_2, epsilon):
self.activation = activation
self.solver = solver
self.alpha = alpha
self.batch_size = batch_size
self.learning_rate = learning_rate
self.learning_rate_init = learning_rate_init
self.power_t = power_t
self.max_iter = max_iter
self.loss = loss
self.hidden_layer_sizes = hidden_layer_sizes
self.shuffle = shuffle
self.random_state = random_state
self.tol = tol
self.verbose = verbose
self.warm_start = warm_start
self.momentum = momentum
self.nesterovs_momentum = nesterovs_momentum
self.early_stopping = early_stopping
self.validation_fraction = validation_fraction
self.beta_1 = beta_1
self.beta_2 = beta_2
self.epsilon = epsilon
def _unpack(self, packed_parameters):
"""Extract the coefficients and intercepts from packed_parameters."""
for i in range(self.n_layers_ - 1):
start, end, shape = self._coef_indptr[i]
self.coefs_[i] = np.reshape(packed_parameters[start:end], shape)
start, end = self._intercept_indptr[i]
self.intercepts_[i] = packed_parameters[start:end]
def _forward_pass(self, activations, training=True):
"""Perform a forward pass on the network by computing the values
of the neurons in the hidden layers and the output layer.
Parameters
----------
activations : list, length = n_layers - 1
The ith element of the list holds the values of the ith layer.
with_output_activation : bool, default True
If True, the output passes through the output activation
function, which is either the softmax function or the
logistic function
"""
hidden_activation = ACTIVATIONS[self.activation]
'''for explanations start'''
# Iterate over the hidden layers
# activated neuron for each hidden layer will be saved here.
# i'th element of the list will be i'th layer's activated neurons
activated_neurons = []
activated_neurons_as_dict = []
activated_neurons_raw_sum = []
activated_neurons_as_np = np.zeros_like((len(activations[0]) * len(activations[0][0])))
'''for explanations end'''
# Iterate over the hidden layers
for i in range(self.n_layers_ - 1):
activations[i + 1] = safe_sparse_dot(activations[i],
self.coefs_[i])
activations[i + 1] += self.intercepts_[i]
# For the hidden layers
if (i + 1) != (self.n_layers_ - 1):
activations[i + 1] = hidden_activation(activations[i + 1])
'''for explanations start'''
# elements of activations are the neurons
# activations are list, and i'th element of the activations activation[i] is
# matrix of dimension = instances * neurons
# activations[0] = layer_1 * neurons_in_layer_1
# iterate over all input_instances
# calculate the values only for test set...
# we have to implement it for train set also
if not training:
# print('inside training')
activated_n = set()
activated_n_as_dict = {}
activated_n_raw_sum = np.zeros(activations[i + 1].shape[1])
# print('layer : ',(i+1))
for input_instance_i in activations[i + 1]:
neuron_set = set(index for index, value in enumerate(
input_instance_i) if value >= np.mean(input_instance_i))
neuron_dict = dict([(index, value) if value >= np.mean(input_instance_i)
else (index, 0) for index, value in enumerate(input_instance_i)])
activated_n_raw_sum += np.array([1 if value >= np.mean(
input_instance_i) else 0 for value in input_instance_i])
activated_n |= neuron_set
# print('neurons: ', neuron)
# print('activated_n: ',activated_n)
# print('')
activated_neurons.append(activated_n)
activated_neurons_raw_sum.append(activated_n_raw_sum)
# print('activated_neurons: ',activated_neurons)
# print('')
# print('')
# print('activations[%s]: ' %(i+1), type(activations[i+1]), activations[i+1].shape,activations[i+1])
'''for explanations end'''
# For the last layer
output_activation = ACTIVATIONS[self.out_activation_]
activations[i + 1] = output_activation(activations[i + 1])
'''for explanations start'''
return activations, activated_neurons, activated_neurons_raw_sum
'''for explanations end'''
def _compute_loss_grad(self, layer, n_samples, activations, deltas,
coef_grads, intercept_grads):
"""Compute the gradient of loss with respect to coefs and intercept for
specified layer.
This function does backpropagation for the specified one layer.
"""
coef_grads[layer] = safe_sparse_dot(activations[layer].T,
deltas[layer])
coef_grads[layer] += (self.alpha * self.coefs_[layer])
coef_grads[layer] /= n_samples
intercept_grads[layer] = np.mean(deltas[layer], 0)
return coef_grads, intercept_grads
def _loss_grad_lbfgs(self, packed_coef_inter, X, y, activations, deltas,
coef_grads, intercept_grads):
"""Compute the MLP loss function and its corresponding derivatives
with respect to the different parameters given in the initialization.
Returned gradients are packed in a single vector so it can be used
in lbfgs
Parameters
----------
packed_parameters : array-like
A vector comprising the flattened coefficients and intercepts.
X : {array-like, sparse matrix}, shape (n_samples, n_features)
The input data.
y : array-like, shape (n_samples,)
The target values.
activations : list, length = n_layers - 1
The ith element of the list holds the values of the ith layer.
deltas : list, length = n_layers - 1
The ith element of the list holds the difference between the
activations of the i + 1 layer and the backpropagated error.
More specifically, deltas are gradients of loss with respect to z
in each layer, where z = wx + b is the value of a particular layer
before passing through the activation function
coef_grad : list, length = n_layers - 1
The ith element contains the amount of change used to update the
coefficient parameters of the ith layer in an iteration.
intercept_grads : list, length = n_layers - 1
The ith element contains the amount of change used to update the
intercept parameters of the ith layer in an iteration.
Returns
-------
loss : float
grad : array-like, shape (number of nodes of all layers,)
"""
self._unpack(packed_coef_inter)
loss, coef_grads, intercept_grads = self._backprop(
X, y, activations, deltas, coef_grads, intercept_grads)
self.n_iter_ += 1
grad = _pack(coef_grads, intercept_grads)
return loss, grad
def _backprop(self, X, y, activations, deltas, coef_grads,
intercept_grads):
"""Compute the MLP loss function and its corresponding derivatives
with respect to each parameter: weights and bias vectors.
Parameters
----------
X : {array-like, sparse matrix}, shape (n_samples, n_features)
The input data.
y : array-like, shape (n_samples,)
The target values.
activations : list, length = n_layers - 1
The ith element of the list holds the values of the ith layer.
deltas : list, length = n_layers - 1
The ith element of the list holds the difference between the
activations of the i + 1 layer and the backpropagated error.
More specifically, deltas are gradients of loss with respect to z
in each layer, where z = wx + b is the value of a particular layer
before passing through the activation function
coef_grad : list, length = n_layers - 1
The ith element contains the amount of change used to update the
coefficient parameters of the ith layer in an iteration.
intercept_grads : list, length = n_layers - 1
The ith element contains the amount of change used to update the
intercept parameters of the ith layer in an iteration.
Returns
-------
loss : float
coef_grads : list, length = n_layers - 1
intercept_grads : list, length = n_layers - 1
"""
n_samples = X.shape[0]
# Forward propagate
'''for explanations start'''
activations, activated_neurons, activated_neurons_raw_sum = self._forward_pass(activations, training=True)
'''for explanations end'''
# Get loss
loss_func_name = self.loss
if loss_func_name == 'log_loss' and self.out_activation_ == 'logistic':
loss_func_name = 'binary_log_loss'
loss = LOSS_FUNCTIONS[loss_func_name](y, activations[-1])
# Add L2 regularization term to loss
values = np.sum(
np.array([np.dot(s.ravel(), s.ravel()) for s in self.coefs_]))
loss += (0.5 * self.alpha) * values / n_samples
# Backward propagate
last = self.n_layers_ - 2
# The calculation of delta[last] here works with following
# combinations of output activation and loss function:
# sigmoid and binary cross entropy, softmax and categorical cross
# entropy, and identity with squared loss
deltas[last] = activations[-1] - y
# Compute gradient for the last layer
coef_grads, intercept_grads = self._compute_loss_grad(
last, n_samples, activations, deltas, coef_grads, intercept_grads)
# Iterate over the hidden layers
for i in range(self.n_layers_ - 2, 0, -1):
deltas[i - 1] = safe_sparse_dot(deltas[i], self.coefs_[i].T)
inplace_derivative = DERIVATIVES[self.activation]
inplace_derivative(activations[i], deltas[i - 1])
coef_grads, intercept_grads = self._compute_loss_grad(
i - 1, n_samples, activations, deltas, coef_grads,
intercept_grads)
# print('loss.len: ', len(loss))
'''for explanations start'''
# for index, activation in enumerate(activations):
# print('activations[%d]' %index, activation[index])
'''for explanations end'''
return loss, coef_grads, intercept_grads
def _initialize(self, y, layer_units):
# set all attributes, allocate weights etc for first call
# Initialize parameters
self.n_iter_ = 0
self.t_ = 0
self.n_outputs_ = y.shape[1]
# Compute the number of layers
self.n_layers_ = len(layer_units)
# Output for regression
if not isinstance(self, ClassifierMixin):
self.out_activation_ = 'identity'
# Output for multi class
elif self._label_binarizer.y_type_ == 'multiclass':
self.out_activation_ = 'softmax'
# Output for binary class and multi-label
else:
self.out_activation_ = 'logistic'
# Initialize coefficient and intercept layers
self.coefs_ = []
self.intercepts_ = []
for i in range(self.n_layers_ - 1):
coef_init, intercept_init = self._init_coef(layer_units[i],
layer_units[i + 1])
self.coefs_.append(coef_init)
self.intercepts_.append(intercept_init)
if self.solver in _STOCHASTIC_SOLVERS:
self.loss_curve_ = []
self._no_improvement_count = 0
if self.early_stopping:
self.validation_scores_ = []
self.best_validation_score_ = -np.inf
else:
self.best_loss_ = np.inf
def _init_coef(self, fan_in, fan_out):
if self.activation == 'logistic':
# Use the initialization method recommended by
# Glorot et al.
init_bound = np.sqrt(2. / (fan_in + fan_out))
elif self.activation in ('identity', 'tanh', 'relu'):
init_bound = np.sqrt(6. / (fan_in + fan_out))
else:
# this was caught earlier, just to make sure
raise ValueError("Unknown activation function %s" %
self.activation)
coef_init = self._random_state.uniform(-init_bound, init_bound,
(fan_in, fan_out))
intercept_init = self._random_state.uniform(-init_bound, init_bound,
fan_out)
return coef_init, intercept_init
def _fit(self, X, y, incremental=False):
# Make sure self.hidden_layer_sizes is a list
hidden_layer_sizes = self.hidden_layer_sizes
if not hasattr(hidden_layer_sizes, "__iter__"):
hidden_layer_sizes = [hidden_layer_sizes]
hidden_layer_sizes = list(hidden_layer_sizes)
# Validate input parameters.
self._validate_hyperparameters()
if np.any(np.array(hidden_layer_sizes) <= 0):
raise ValueError("hidden_layer_sizes must be > 0, got %s." %
hidden_layer_sizes)
X, y = self._validate_input(X, y, incremental)
n_samples, n_features = X.shape
# Ensure y is 2D
if y.ndim == 1:
y = y.reshape((-1, 1))
self.n_outputs_ = y.shape[1]
layer_units = ([n_features] + hidden_layer_sizes +
[self.n_outputs_])
# check random state
self._random_state = check_random_state(self.random_state)
if not hasattr(self, 'coefs_') or (not self.warm_start and not
incremental):
# First time training the model
self._initialize(y, layer_units)
# lbfgs does not support mini-batches
if self.solver == 'lbfgs':
batch_size = n_samples
elif self.batch_size == 'auto':
batch_size = min(200, n_samples)
else:
if self.batch_size < 1 or self.batch_size > n_samples:
warnings.warn("Got `batch_size` less than 1 or larger than "
"sample size. It is going to be clipped")
batch_size = np.clip(self.batch_size, 1, n_samples)
# Initialize lists
activations = [X]
activations.extend(np.empty((batch_size, n_fan_out))
for n_fan_out in layer_units[1:])
deltas = [np.empty_like(a_layer) for a_layer in activations]
# coef is weight matrix
coef_grads = [np.empty((n_fan_in_, n_fan_out_)) for n_fan_in_,
n_fan_out_ in zip(layer_units[:-1],
layer_units[1:])]
# intercept is bias
intercept_grads = [np.empty(n_fan_out_) for n_fan_out_ in
layer_units[1:]]
# Run the Stochastic optimization solver
if self.solver in _STOCHASTIC_SOLVERS:
'''for explanation start'''
activations_over_all_itr = self._fit_stochastic(X, y, activations, deltas, coef_grads,
intercept_grads, layer_units, incremental)
'''for explanation end'''
# Run the LBFGS solver
elif self.solver == 'lbfgs':
self._fit_lbfgs(X, y, activations, deltas, coef_grads,
intercept_grads, layer_units)
''' for explanation start'''
return self, activations_over_all_itr
''' for explanation end'''
def _validate_hyperparameters(self):
if not isinstance(self.shuffle, bool):
raise ValueError("shuffle must be either True or False, got %s." %
self.shuffle)
if self.max_iter <= 0:
raise ValueError("max_iter must be > 0, got %s." % self.max_iter)
if self.alpha < 0.0:
raise ValueError("alpha must be >= 0, got %s." % self.alpha)
if (self.learning_rate in ["constant", "invscaling", "adaptive"] and
self.learning_rate_init <= 0.0):
raise ValueError("learning_rate_init must be > 0, got %s." %
self.learning_rate)
if self.momentum > 1 or self.momentum < 0:
raise ValueError("momentum must be >= 0 and <= 1, got %s" %
self.momentum)
if not isinstance(self.nesterovs_momentum, bool):
raise ValueError("nesterovs_momentum must be either True or False,"
" got %s." % self.nesterovs_momentum)
if not isinstance(self.early_stopping, bool):
raise ValueError("early_stopping must be either True or False,"
" got %s." % self.early_stopping)
if self.validation_fraction < 0 or self.validation_fraction >= 1:
raise ValueError("validation_fraction must be >= 0 and < 1, "
"got %s" % self.validation_fraction)
if self.beta_1 < 0 or self.beta_1 >= 1:
raise ValueError("beta_1 must be >= 0 and < 1, got %s" %
self.beta_1)
if self.beta_2 < 0 or self.beta_2 >= 1:
raise ValueError("beta_2 must be >= 0 and < 1, got %s" %
self.beta_2)
if self.epsilon <= 0.0:
raise ValueError("epsilon must be > 0, got %s." % self.epsilon)
# raise ValueError if not registered
supported_activations = ('identity', 'logistic', 'tanh', 'relu')
if self.activation not in supported_activations:
raise ValueError("The activation '%s' is not supported. Supported "
"activations are %s." % (self.activation,
supported_activations))
if self.learning_rate not in ["constant", "invscaling", "adaptive"]:
raise ValueError("learning rate %s is not supported. " %
self.learning_rate)
supported_solvers = _STOCHASTIC_SOLVERS + ["lbfgs"]
if self.solver not in supported_solvers:
raise ValueError("The solver %s is not supported. "
" Expected one of: %s" %
(self.solver, ", ".join(supported_solvers)))
def _fit_lbfgs(self, X, y, activations, deltas, coef_grads,
intercept_grads, layer_units):
# Store meta information for the parameters
self._coef_indptr = []
self._intercept_indptr = []
start = 0
# Save sizes and indices of coefficients for faster unpacking
for i in range(self.n_layers_ - 1):
n_fan_in, n_fan_out = layer_units[i], layer_units[i + 1]
end = start + (n_fan_in * n_fan_out)
self._coef_indptr.append((start, end, (n_fan_in, n_fan_out)))
start = end
# Save sizes and indices of intercepts for faster unpacking
for i in range(self.n_layers_ - 1):
end = start + layer_units[i + 1]
self._intercept_indptr.append((start, end))
start = end
# Run LBFGS
packed_coef_inter = _pack(self.coefs_,
self.intercepts_)
if self.verbose is True or self.verbose >= 1:
iprint = 1
else:
iprint = -1
optimal_parameters, self.loss_, d = fmin_l_bfgs_b(
x0=packed_coef_inter,
func=self._loss_grad_lbfgs,
maxfun=self.max_iter,
iprint=iprint,
pgtol=self.tol,
args=(X, y, activations, deltas, coef_grads, intercept_grads))
self._unpack(optimal_parameters)
def _fit_stochastic(self, X, y, activations, deltas, coef_grads,
intercept_grads, layer_units, incremental):
if not incremental or not hasattr(self, '_optimizer'):
params = self.coefs_ + self.intercepts_
if self.solver == 'sgd':
self._optimizer = SGDOptimizer(
params, self.learning_rate_init, self.learning_rate,
self.momentum, self.nesterovs_momentum, self.power_t)
elif self.solver == 'adam':
self._optimizer = AdamOptimizer(
params, self.learning_rate_init, self.beta_1, self.beta_2,
self.epsilon)
# early_stopping in partial_fit doesn't make sense
early_stopping = self.early_stopping and not incremental
if early_stopping:
X, X_val, y, y_val = train_test_split(
X, y, random_state=self._random_state,
test_size=self.validation_fraction)
if isinstance(self, ClassifierMixin):
y_val = self._label_binarizer.inverse_transform(y_val)
else:
X_val = None
y_val = None
n_samples = X.shape[0]
if self.batch_size == 'auto':
batch_size = min(200, n_samples)
else:
batch_size = np.clip(self.batch_size, 1, n_samples)
'''for explanation start'''
activations_over_all_itr = []
'''for explanation end'''
try:
for it in range(self.max_iter):
X, y = shuffle(X, y, random_state=self._random_state)
accumulated_loss = 0.0
for batch_slice in gen_batches(n_samples, batch_size):
activations[0] = X[batch_slice]
batch_loss, coef_grads, intercept_grads = self._backprop(
X[batch_slice], y[batch_slice], activations, deltas,
coef_grads, intercept_grads)
accumulated_loss += batch_loss * (batch_slice.stop -
batch_slice.start)
# update weights
grads = coef_grads + intercept_grads
self._optimizer.update_params(grads)
'''for explanation start'''
activations_over_all_itr.append(activations)
'''for explanation end'''
self.n_iter_ += 1
self.loss_ = accumulated_loss / X.shape[0]
self.t_ += n_samples
self.loss_curve_.append(self.loss_)
if self.verbose:
print("Iteration %d, loss = %.8f" % (self.n_iter_,
self.loss_))
# update no_improvement_count based on training loss or
# validation score according to early_stopping
self._update_no_improvement_count(early_stopping, X_val, y_val)
# for learning rate that needs to be updated at iteration end
self._optimizer.iteration_ends(self.t_)
if self._no_improvement_count > 2:
# not better than last two iterations by tol.
# stop or decrease learning rate
if early_stopping:
msg = ("Validation score did not improve more than "
"tol=%f for two consecutive epochs." % self.tol)
else:
msg = ("Training loss did not improve more than tol=%f"
" for two consecutive epochs." % self.tol)
is_stopping = self._optimizer.trigger_stopping(
msg, self.verbose)
if is_stopping:
break
else:
self._no_improvement_count = 0
if incremental:
break
if self.n_iter_ == self.max_iter:
warnings.warn(
"Stochastic Optimizer: Maximum iterations (%d) "
"reached and the optimization hasn't converged yet."
% self.max_iter, ConvergenceWarning)
except KeyboardInterrupt:
warnings.warn("Training interrupted by user.")
if early_stopping:
# restore best weights
self.coefs_ = self._best_coefs
self.intercepts_ = self._best_intercepts
'''for explanation start'''
return activations_over_all_itr
'''for explanation end'''
def _update_no_improvement_count(self, early_stopping, X_val, y_val):
if early_stopping:
# compute validation score, use that for stopping
self.validation_scores_.append(self.score(X_val, y_val))
if self.verbose:
print("Validation score: %f" % self.validation_scores_[-1])
# update best parameters
# use validation_scores_, not loss_curve_
# let's hope no-one overloads .score with mse
last_valid_score = self.validation_scores_[-1]
if last_valid_score < (self.best_validation_score_ +
self.tol):
self._no_improvement_count += 1
else:
self._no_improvement_count = 0
if last_valid_score > self.best_validation_score_:
self.best_validation_score_ = last_valid_score
self._best_coefs = [c.copy() for c in self.coefs_]
self._best_intercepts = [i.copy()
for i in self.intercepts_]
else:
if self.loss_curve_[-1] > self.best_loss_ - self.tol:
self._no_improvement_count += 1
else:
self._no_improvement_count = 0
if self.loss_curve_[-1] < self.best_loss_:
self.best_loss_ = self.loss_curve_[-1]
def fit(self, X, y):
"""Fit the model to data matrix X and target(s) y.
Parameters
----------
X : array-like or sparse matrix, shape (n_samples, n_features)
The input data.
y : array-like, shape (n_samples,) or (n_samples, n_outputs)
The target values (class labels in classification, real numbers in
regression).
Returns
-------
self : returns a trained MLP model.
"""
return self._fit(X, y, incremental=False)
@property
def partial_fit(self):
"""Fit the model to data matrix X and target y.
Parameters
----------
X : {array-like, sparse matrix}, shape (n_samples, n_features)
The input data.
y : array-like, shape (n_samples,)
The target values.
Returns
-------
self : returns a trained MLP model.
"""
if self.solver not in _STOCHASTIC_SOLVERS:
raise AttributeError("partial_fit is only available for stochastic"
" optimizers. %s is not stochastic."
% self.solver)
return self._partial_fit
def _partial_fit(self, X, y, classes=None):
return self._fit(X, y, incremental=True)
def _predict(self, X):
"""Predict using the trained model
Parameters
----------
X : {array-like, sparse matrix}, shape (n_samples, n_features)
The input data.
Returns
-------
y_pred : array-like, shape (n_samples,) or (n_samples, n_outputs)
The decision function of the samples for each class in the model.
"""
X = check_array(X, accept_sparse=['csr', 'csc', 'coo'])
# Make sure self.hidden_layer_sizes is a list
hidden_layer_sizes = self.hidden_layer_sizes
if not hasattr(hidden_layer_sizes, "__iter__"):
hidden_layer_sizes = [hidden_layer_sizes]
hidden_layer_sizes = list(hidden_layer_sizes)
layer_units = [X.shape[1]] + hidden_layer_sizes + \
[self.n_outputs_]
# Initialize layers
# now activations is simply X or all input instances
activations = [X]
for i in range(self.n_layers_ - 1):
m = np.empty((X.shape[0], layer_units[i + 1]))
activations.append(m)
# forward propagate
'''for explanations start'''
activations, activated_neurons, activated_neurons_raw_sum = self._forward_pass(activations, training=False)
'''for explanations end'''
y_pred = activations[-1]
'''for explanations start'''
return y_pred, activations, activated_neurons, activated_neurons_raw_sum
'''for explanations end'''
def _fit_stochastic(
self,
X,
y,
activations,
deltas,
coef_grads,
intercept_grads,
layer_units,
incremental,
):
if not incremental or not hasattr(self, "_optimizer"):
params = self.coefs_ + self.intercepts_
if self.solver == "sgd":
self._optimizer = SGDOptimizer(
params,
self.learning_rate_init,
self.learning_rate,
self.momentum,
self.nesterovs_momentum,
self.power_t,
)
elif self.solver == "adam":
self._optimizer = AdamOptimizer(
params,
self.learning_rate_init,
self.beta_1,
self.beta_2,
self.epsilon,
)
# early_stopping in partial_fit doesn't make sense
early_stopping = self.early_stopping and not incremental
if early_stopping:
# don't stratify in multilabel classification
# should_stratify = is_classifier(self) and self.n_outputs_ == 1
# stratify = y if should_stratify else None
# X, X_val, y, y_val = train_test_split(
# X,
# y,
# random_state=self._random_state,
# test_size=self.validation_fraction,
# stratify=stratify,
# )
# if is_classifier(self):
# y_val = self._label_binarizer.inverse_transform(y_val)
# --------------------------- #
# Custom validation set #
# --------------------------- #
X_val = self.custom_validation_data[0]
y_val = self.custom_validation_data[1]
if type(X_val) is not np.ndarray:
X_val = X_val.to_numpy()
if type(y_val) is not np.ndarray:
y_val = y_val.to_numpy()
# --------------------------- #
# Custom code end #
# --------------------------- #
else:
X_val = None
y_val = None
n_samples = X.shape[0]
if self.batch_size == "auto":
batch_size = min(200, n_samples)
else:
batch_size = np.clip(self.batch_size, 1, n_samples)
try:
for it in tqdm(range(self.max_iter), unit="iteration"):
if self.shuffle:
X, y = shuffle(X, y, random_state=self._random_state)
accumulated_loss = 0.0
for batch_slice in gen_batches(n_samples, batch_size):
activations[0] = X[batch_slice]
batch_loss, coef_grads, intercept_grads = self._backprop(
X[batch_slice],
y[batch_slice],
activations,
deltas,
coef_grads,
intercept_grads,
)
accumulated_loss += batch_loss * (batch_slice.stop -
batch_slice.start)
# update weights
grads = coef_grads + intercept_grads
self._optimizer.update_params(grads)
self.n_iter_ += 1
self.loss_ = accumulated_loss / X.shape[0]
self.t_ += n_samples
self.loss_curve_.append(self.loss_)
if self.verbose:
print("Iteration %d, loss = %.8f" %
(self.n_iter_, self.loss_))
# update no_improvement_count based on training loss or
# validation score according to early_stopping
self._update_no_improvement_count(early_stopping, X_val, y_val)
# for learning rate that needs to be updated at iteration end
self._optimizer.iteration_ends(self.t_)
if self._no_improvement_count > self.n_iter_no_change:
# not better than last `n_iter_no_change` iterations by tol
# stop or decrease learning rate
if early_stopping:
msg = ("Validation score did not improve more than "
"tol=%f for %d consecutive epochs." %
(self.tol, self.n_iter_no_change))
else:
msg = ("Training loss did not improve more than tol=%f"
" for %d consecutive epochs." %
(self.tol, self.n_iter_no_change))
is_stopping = self._optimizer.trigger_stopping(
msg, self.verbose)
if is_stopping:
break
else:
self._no_improvement_count = 0
if incremental:
break
if self.n_iter_ == self.max_iter:
warnings.warn(
"Stochastic Optimizer: Maximum iterations (%d) "
"reached and the optimization hasn't converged yet." %
self.max_iter,
ConvergenceWarning,
)
except KeyboardInterrupt:
warnings.warn("Training interrupted by user.")
if early_stopping:
# restore best weights
self.coefs_ = self._best_coefs
self.intercepts_ = self._best_intercepts
class ModifiedMLPClassifier(MLPClassifier):
def __init__(
self,
hidden_layer_sizes=(100, ),
activation="relu",
solver="adam",
alpha=0.0001,
batch_size="auto",
learning_rate="constant",
learning_rate_init=0.001,
power_t=0.5,
max_iter=200,
shuffle=True,
random_state=None,
tol=1e-4,
verbose=False,
warm_start=False,
momentum=0.9,
nesterovs_momentum=True,
early_stopping=False,
validation_fraction=0.1,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-8,
n_iter_no_change=10,
max_fun=15000,
custom_validation_data=None,
):
super().__init__(
hidden_layer_sizes=hidden_layer_sizes,
activation=activation,
solver=solver,
alpha=alpha,
batch_size=batch_size,
learning_rate=learning_rate,
learning_rate_init=learning_rate_init,
power_t=power_t,
max_iter=max_iter,
shuffle=shuffle,
random_state=random_state,
tol=tol,
verbose=verbose,
warm_start=warm_start,
momentum=momentum,
nesterovs_momentum=nesterovs_momentum,
early_stopping=early_stopping,
validation_fraction=validation_fraction,
beta_1=beta_1,
beta_2=beta_2,
epsilon=epsilon,
n_iter_no_change=n_iter_no_change,
max_fun=max_fun,
)
self.custom_validation_data = custom_validation_data
def score(self, X, y, sample_weight=None):
"""
Return the LRAP on the given test data and labels.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Test samples.
y : array-like of shape (n_samples, n_outputs)
True labels for X.
sample_weight : array-like of shape (n_samples,), default=None
Sample weights.
Returns
-------
score : float
LRAP of self.predict_proba(X) wrt. y.
"""
return LRAP(y, self._predict(X), sample_weight=sample_weight)
def _fit_stochastic(
self,
X,
y,
activations,
deltas,
coef_grads,
intercept_grads,
layer_units,
incremental,
):
if not incremental or not hasattr(self, "_optimizer"):
params = self.coefs_ + self.intercepts_
if self.solver == "sgd":
self._optimizer = SGDOptimizer(
params,
self.learning_rate_init,
self.learning_rate,
self.momentum,
self.nesterovs_momentum,
self.power_t,
)
elif self.solver == "adam":
self._optimizer = AdamOptimizer(
params,
self.learning_rate_init,
self.beta_1,
self.beta_2,
self.epsilon,
)
# early_stopping in partial_fit doesn't make sense
early_stopping = self.early_stopping and not incremental
if early_stopping:
# don't stratify in multilabel classification
# should_stratify = is_classifier(self) and self.n_outputs_ == 1
# stratify = y if should_stratify else None
# X, X_val, y, y_val = train_test_split(
# X,
# y,
# random_state=self._random_state,
# test_size=self.validation_fraction,
# stratify=stratify,
# )
# if is_classifier(self):
# y_val = self._label_binarizer.inverse_transform(y_val)
# --------------------------- #
# Custom validation set #
# --------------------------- #
X_val = self.custom_validation_data[0]
y_val = self.custom_validation_data[1]
if type(X_val) is not np.ndarray:
X_val = X_val.to_numpy()
if type(y_val) is not np.ndarray:
y_val = y_val.to_numpy()
# --------------------------- #
# Custom code end #
# --------------------------- #
else:
X_val = None
y_val = None
n_samples = X.shape[0]
if self.batch_size == "auto":
batch_size = min(200, n_samples)
else:
batch_size = np.clip(self.batch_size, 1, n_samples)
try:
for it in tqdm(range(self.max_iter), unit="iteration"):
if self.shuffle:
X, y = shuffle(X, y, random_state=self._random_state)
accumulated_loss = 0.0
for batch_slice in gen_batches(n_samples, batch_size):
activations[0] = X[batch_slice]
batch_loss, coef_grads, intercept_grads = self._backprop(
X[batch_slice],
y[batch_slice],
activations,
deltas,
coef_grads,
intercept_grads,
)
accumulated_loss += batch_loss * (batch_slice.stop -
batch_slice.start)
# update weights
grads = coef_grads + intercept_grads
self._optimizer.update_params(grads)
self.n_iter_ += 1
self.loss_ = accumulated_loss / X.shape[0]
self.t_ += n_samples
self.loss_curve_.append(self.loss_)
if self.verbose:
print("Iteration %d, loss = %.8f" %
(self.n_iter_, self.loss_))
# update no_improvement_count based on training loss or
# validation score according to early_stopping
self._update_no_improvement_count(early_stopping, X_val, y_val)
# for learning rate that needs to be updated at iteration end
self._optimizer.iteration_ends(self.t_)
if self._no_improvement_count > self.n_iter_no_change:
# not better than last `n_iter_no_change` iterations by tol
# stop or decrease learning rate
if early_stopping:
msg = ("Validation score did not improve more than "
"tol=%f for %d consecutive epochs." %
(self.tol, self.n_iter_no_change))
else:
msg = ("Training loss did not improve more than tol=%f"
" for %d consecutive epochs." %
(self.tol, self.n_iter_no_change))
is_stopping = self._optimizer.trigger_stopping(
msg, self.verbose)
if is_stopping:
break
else:
self._no_improvement_count = 0
if incremental:
break
if self.n_iter_ == self.max_iter:
warnings.warn(
"Stochastic Optimizer: Maximum iterations (%d) "
"reached and the optimization hasn't converged yet." %
self.max_iter,
ConvergenceWarning,
)
except KeyboardInterrupt:
warnings.warn("Training interrupted by user.")
if early_stopping:
# restore best weights
self.coefs_ = self._best_coefs
self.intercepts_ = self._best_intercepts