def transform(self, X, n_components=None, in_place=False):
""" Project a (new) dataset onto the components.
Return the projected data and the associated d.
We have to recompute U and d because the argument may not have the same
number of lines.
The argument must have the same number of columns than the datset used
to fit the estimator.
"""
if n_components is None:
n_components = self.n_components
Xk = check_arrays(X)
if not in_place:
Xk = Xk.copy()
n, p = Xk.shape
if p != self.V.shape[0]:
raise ValueError("The argument must have the same number of "
"columns than the datset used to fit the "
"estimator.")
U = np.zeros((n, n_components))
d = np.zeros((n_components, ))
for k in range(n_components):
# Project on component j
vk = self.V[:, k].reshape(-1, 1)
uk = np.dot(X, vk)
uk /= np.linalg.norm(uk)
U[:, k] = uk[:, 0]
dk = self.compute_d(Xk, uk, vk)
d[k] = dk
# Residualize
Xk -= dk * np.dot(uk, vk.T)
return U, d
def l1_max(cls, X):
"""Return the maximal value for l1 parameter for a given data matrix
"""
X = check_arrays(X)
s = [np.linalg.norm(X[:, i]) for i in range(X.shape[1])]
l1_max = np.max(s) / X.shape[0]
return l1_max
def __init__(self, X, y, k=0.0, weights=None, penalty_start=0, mean=True):
"""
Parameters
----------
X : Numpy array (n-by-p). The regressor matrix. Training vectors, where
n is the number of samples and p is the number of features.
y : Numpy array (n-by-1). The regressand vector. Target values (class
labels in classification).
k : Non-negative float. The ridge parameter.
weights: Numpy array (n-by-1). The sample's weights.
penalty_start : Non-negative integer. The number of columns, variables
etc., to except from penalisation. Equivalently, the first
index to be penalised. Default is 0, all columns are included.
mean : Boolean. Whether to compute the mean loss or not. Default is
True, the mean loss is computed.
"""
self.X = X
self.y = y
self.k = np.maximum(0.0, check_arrays(k, flatten=True))
if weights is None:
weights = np.ones(y.shape) # .reshape(y.shape)
self.weights = weights
self.penalty_start = max(0, int(penalty_start))
self.mean = bool(mean)
self.reset()
def __init__(self, X, y, k, penalty_start=0, mean=True):
"""
Parameters
----------
X : Numpy array (n-by-p). The regressor matrix.
y : Numpy array (n-by-1). The regressand vector.
k : Non-negative float. The ridge parameter.
penalty_start : Non-negative integer. The number of columns, variables
etc., to except from penalisation. Equivalently, the first
index to be penalised. Default is 0, all columns are included.
mean : Boolean. Whether to compute the squared loss or the mean
squared loss. Default is True, the mean squared loss.
"""
self.X = X
self.y = y
self.k = np.maximum(0.0, check_arrays(k, flatten=True))
self.penalty_start = max(0, int(penalty_start))
self.mean = bool(mean)
self.reset()
def l1_max_linear_loss(X, y, mean=True):
"""Estimate class weights for unbalanced datasets.
Parameters
----------
X : array-like, shape (n_samples, n_features)
Array of input data;
y : array-like, shape (n_samples, 1)
Target values
mean : Boolean. Whether to compute the squared loss or the mean squared
loss. Default is True, the mean squared loss.
Returns
-------
l1_max : scalar
Maximum l1 pentlty to avoid null solution
Example
-------
"""
X, y = check_arrays(X, y)
n = float(X.shape[0])
scale = 1.0 / n if mean else 1.
l1_max = scale * np.abs(np.dot(X.T, y)).max()
return 0.95 * l1_max
def transform(X, V):
""" Project a (new) dataset onto the components.
Return the projected data and the associated d.
We have to recompute U and d because the argument may not have the same
number of lines.
The argument must have the same number of columns than the datset used
to fit the estimator.
"""
Xk = check_arrays(X)
Xk = Xk.copy()
n, p = Xk.shape
if p != V.shape[0]:
raise ValueError("The argument must have the same number of "
"columns than the datset used to fit the "
"estimator.")
U = np.zeros((n, V.shape[1]))
d = np.zeros((V.shape[1], ))
for k in range(V.shape[1]):
# Project on component j
vk = V[:, k].reshape(-1, 1)
uk = np.dot(X, vk)
uk /= np.linalg.norm(uk)
U[:, k] = uk[:, 0]
dk = compute_d(X, uk, vk)
d[k] = dk
# Residualize
Xk -= dk * np.dot(uk, vk.T)
return U, d
def __init__(self, X, y, output_nodes, loss,
step_size=step_sizes.ConstantStepSize(0.01), weights=None):
X, y = check_arrays(X, y)
self.X = X
self.y = y
self._input = InputLayer(num_nodes=X.shape[1])
self._layers = []
if self._all_nodes_equal(output_nodes, SoftmaxNode) \
and isinstance(loss, CategoricalCrossEntropyLoss):
self._output = SoftmaxCategoricalCrossEntropyOutputLayer(loss,
num_nodes=y.shape[1],
nodes=output_nodes,
weights=weights)
else:
self._output = OutputLayer(loss,
num_nodes=y.shape[1],
nodes=output_nodes,
weights=weights)
loss.set_target(y.T) # Note: The network outputs column vectors
self._step_size = step_size
self.reset()
def l1_max(cls, X):
"""Return the maximal value for l1 parameter for a given data matrix
"""
X = check_arrays(X)
s = np.sqrt((X**2).sum(axis=0))
#s_old = np.asarray([np.linalg.norm(X[:, i]) for i in range(X.shape[1])])
#assert np.allclose(s, s_old)
l1_max = np.max(s) / X.shape[0]
return l1_max
def _vector(self, x1, x2):
x1, x2 = check_arrays(x1, x2)
val = np.dot(x1.T, x2)
if isinstance(val, np.ndarray):
val = val[0, 0]
return val
def _vector(self, x1, x2):
x1, x2 = check_arrays(x1, x2)
val = np.dot(x1.T, x2)
if isinstance(val, np.ndarray):
val = val[0, 0]
return val
def wilcoxon_test(x, Y, zero_method="zsplit", correction=False):
"""Performs the Wilcoxon signed rank test for comparing one classifier
to several other classifiers.
It tests the null hypothesis that two related paired samples comes from the
same distribution. It is a non-parametric version of the paired t-test.
Parameters
----------
x : numpy array of shape (n, 1)
The measurements for a single classifier.
Y : numpy array of shape (n, k)
The measurements for k other classifiers.
zero_method : string, {"pratt", "wilcox", "zsplit"}, optional
How to treat zero-differences in the ranking. Default is "zsplit",
splitting the zero-ranks between the positive and negative ranks.
See scipy.stats.wilcoxon for more details.
correction : bool, optional
Whether or not to apply continuity correction by adjusting the rank
statistic by 0.5 towards the mean. Default is False.
Returns
-------
statistics : list of float
The sum of the ranks of the differences, for each of the k classifiers.
p_values : list of float
The two-sided p-values for the tests.
"""
x, Y = check_arrays(x, Y)
if zero_method not in ["pratt", "wilcox", "zsplit"]:
raise ValueError('zero_method must be in ["pratt", "wilcox", '
'"zsplit"].')
correction = bool(correction)
[n, k] = Y.shape
statistics = [0] * k
p_values = [0] * k
for i in range(k):
statistics[i], p_values[i] = stat.wilcoxon(x,
Y[:, i],
zero_method=zero_method,
correction=correction)
return statistics, p_values
def wilcoxon_test(x, Y, zero_method="zsplit", correction=False):
"""Performs the Wilcoxon signed rank test for comparing one classifier
to several other classifiers.
It tests the null hypothesis that two related paired samples comes from the
same distribution. It is a non-parametric version of the paired t-test.
Parameters
----------
x : numpy array of shape (n, 1)
The measurements for a single classifier.
Y : numpy array of shape (n, k)
The measurements for k other classifiers.
zero_method : string, {"pratt", "wilcox", "zsplit"}, optional
How to treat zero-differences in the ranking. Default is "zsplit",
splitting the zero-ranks between the positive and negative ranks.
See scipy.stats.wilcoxon for more details.
correction : bool, optional
Whether or not to apply continuity correction by adjusting the rank
statistic by 0.5 towards the mean. Default is False.
Returns
-------
statistics : list of float
The sum of the ranks of the differences, for each of the k classifiers.
p_values : list of float
The two-sided p-values for the tests.
"""
x, Y = check_arrays(x, Y)
if zero_method not in ["pratt", "wilcox", "zsplit"]:
raise ValueError('zero_method must be in ["pratt", "wilcox", '
'"zsplit"].')
correction = bool(correction)
[n, k] = Y.shape
statistics = [0] * k
p_values = [0] * k
for i in range(k):
statistics[i], p_values[i] = stat.wilcoxon(x, Y[:, i],
zero_method=zero_method,
correction=correction)
return statistics, p_values
def l1_max_logistic_loss(X, y, mean=True, class_weight=None):
"""Estimate class weights for unbalanced datasets.
Parameters
----------
X : array-like, shape (n_samples, n_features)
Array of input data;
y : array-like, shape (n_samples, 1)
Array of class labels in {0, 1};
mean : Boolean. Whether to compute the squared loss or the mean squared
loss. Default is True, the mean squared loss.
class_weight : Dict, 'auto' or None. If 'auto', class weights will be
given inverse proportional to the frequency of the class in
the data. If a dictionary is given, keys are classes and values
are corresponding class weights. If None is given, the class
weights will be uniform.
Returns
-------
l1_max : scalar
Maximum l1 pentlty to avoid null solution
Example
-------
"""
X, y = check_arrays(X, check_labels(y))
sample_weight = class_weight_to_sample_weight(class_weight, y)
y, sample_weight = check_arrays(y, sample_weight)
n = float(X.shape[0])
scale = 1.0 / n if mean else 1.
l1_max = scale * np.abs(np.dot(X.T, sample_weight * (y - 0.5))).max()
return 0.95 * l1_max
def predict(X, V):
""" Return the approximated matrix for a given matrix.
We have to recompute U and d because the argument may not have the same
number of lines.
The argument must have the same number of columns than the datset used
to fit the estimator.
"""
Xk = check_arrays(X)
n, p = Xk.shape
if p != V.shape[0]:
raise ValueError("The argument must have the same number of "
"columns than the datset used to fit the "
"estimator.")
Ut, dt = transform(Xk, V)
Xt = np.zeros(Xk.shape)
for k in range(V.shape[1]):
vk = V[:, k].reshape(-1, 1)
uk = Ut[:, k].reshape(-1, 1)
Xt += compute_rank1_approx(dt[k], uk, vk)
return Xt
def predict(self, X, n_components=None):
""" Return the approximated matrix for a given matrix.
We have to recompute U and d because the argument may not have the same
number of lines.
The argument must have the same number of columns than the datset used
to fit the estimator.
"""
if n_components is None:
n_components = self.n_components
Xk = check_arrays(X)
n, p = Xk.shape
if p != self.V.shape[0]:
raise ValueError("The argument must have the same number of "
"columns than the datset used to fit the "
"estimator.")
Ut, dt = self.transform(Xk, n_components=n_components)
Xt = np.zeros(Xk.shape)
for k in range(n_components):
vk = self.V[:, k].reshape(-1, 1)
uk = Ut[:, k].reshape(-1, 1)
Xt += self.compute_rank1_approx(dt[k], uk, vk)
return Xt
def __init__(self, l, A=None, mu=consts.TOLERANCE, penalty_start=0):
"""
Parameters
----------
l : Non-negative float. The Lagrange multiplier, or regularisation
constant, of the function.
A : A (usually sparse) array. The linear operator for the Nesterov
formulation. May not be None!
mu: Non-negative float. The regularisation constant for the smoothing.
penalty_start : Non-negative integer. The number of columns, variables
etc., to except from penalisation. Equivalently, the first
index to be penalised. Default is 0, all columns are included.
"""
self.l = np.maximum(0.0, check_arrays(l, flatten=True))
if A is None:
raise ValueError("The linear operator A must not be None.")
self._A = A
self.mu = np.asarray(np.maximum(0.0, np.asarray(mu, dtype=float)))
self.penalty_start = max(0, int(penalty_start))
self._alpha = None
def run(self, X, y, alpha=None):
"""Find the best separating margin for the samples in X.
Parameters
----------
X : ndarray
The matrix with samples to separate.
y : array_like
The class belongings for the samples in X. Values must be -1
or 1.
alpha : numpy array
A start vector for the lagrange multipliers. Default is to use a
zero vector.
"""
X, y = check_arrays(X, check_array_in(y, [-1, 1]))
if self.info_requested(Info.ok):
self.info_set(Info.ok, False)
if self.info_requested(Info.time):
_t = utils.time()
if self.info_requested(Info.func_val):
self._f = []
n, p = X.shape
# Set up error cache
self._E = np.zeros(n)
self._Evalid = np.zeros(n, dtype=np.bool)
# Bias (intercept/threshold)
self.bias = 0.0
# Lagrange multipliers
if alpha is None:
self.alpha = np.zeros((n, 1))
else:
self.alpha = alpha
numChanged = 0
examineAll = True
while numChanged > 0 or examineAll:
numChanged = 0
if examineAll:
for i in range(n):
numChanged += self._examineSample(i, X, y)
else:
for i in range(n):
if self.alpha[i, 0] > 0.0 and self.alpha[i, 0] < self.C:
numChanged += self._examineSample(i, X, y)
if examineAll:
examineAll = False
elif numChanged == 0:
examineAll = True
if self.info_requested(Info.time):
self.info_set(Info.time, utils.time() - _t)
if self.info_requested(Info.func_val):
self.info_set(Info.func_val, self._f)
del self._f # Remove for future runs
if self.info_requested(Info.ok):
self.info_set(Info.ok, True)
return self._compute_w(X, y)
def fit(self, X, in_place=False):
"""Fit the estimator to the data
"""
X = check_arrays(X)
l1_max = self.l1_max(X)
if self.l1 >= l1_max:
msg_fmt = "l1 ({l1}) is larger than l1_max ({l1_max})."
msg = msg_fmt.format(l1=self.l1, l1_max=l1_max)
if self.raise_if_l1_too_large:
raise ValueError(msg)
else:
msg += "Solution will be 0."
warnings.warn(msg)
if not in_place:
X = X.copy()
n, p = X.shape
self.U = np.empty([n, self.n_components])
self.d = np.empty([self.n_components])
self.V = np.empty([p, self.n_components])
self.info = []
self.crit = []
self.subfunc = []
self.func = []
self.v_func = []
self.converged = []
for j in range(self.n_components):
if self.verbose:
print('Component', j)
_converged = False
_stopped = False
k = 0
v_new = self.start_vector.get_vector(p)
v_new = v_new / np.linalg.norm(v_new)
_info = []
_crit = []
_subfunc = []
# Real objective function (only if criteria == "Frobenius")
_func = []
# Objective function for fixed u problem
_v_func = []
while (not _converged) and (not _stopped):
if self.verbose:
print('Iteration', k)
# Save previous iteration results
k = k + 1
v_old = np.copy(v_new)
if k >= 2:
u_old = np.copy(u_new)
# Minimize u for v fixed (explicit formula)
u_new = np.dot(X, v_old)
u_new_norm = np.linalg.norm(u_new)
if u_new_norm != 0:
u_new /= u_new_norm
# Minimize v for u fixed
info_conf = [Info.fvalue, Info.num_iter]
function = RightSingularL1L2SmoothedTV_CONESTA(X,
u_new,
self.l1,
self.l2,
self.ltv,
Atv=self.Atv)
algorithm = proximal.CONESTA(eps=self.inner_eps,
tau=self.tau,
max_iter=self.inner_max_iter,
info=info_conf)
v_new = algorithm.run(function, v_old)
info = algorithm.info_get()
_info.append(info)
_v_func.append(function.f(v_new))
if self.verbose:
#print("Inner FISTA iterations:", algorithm.num_iter)
print("Inner calls to FISTA:", info[Info.num_iter])
if self.criterion == "frobenius":
# Compute the normalized Frobenius norm of the
# approximation error.
# X is the deflated matrix so this is the distance
# explained by the current component
d_tmp = self.compute_d(X, u_new, v_new)
# X_approx = d * np.dot(u_new, v_new.T)
X_approx = self.compute_rank1_approx(d_tmp, u_new, v_new)
f = np.linalg.norm(X - X_approx, ord='fro') / (2 * n)
_func.append(f)
del X_approx
# To stop or not to stop
if k >= self.max_iter:
_stopped = True
if k >= 2:
# Evaluate stopping criteria
if self.criterion == "uv":
old = np.vstack((u_old, v_old))
new = np.vstack((u_new, v_new))
crit = np.linalg.norm(new - old)
if self.criterion == "frobenius":
crit = np.abs((_func[-1] - _func[-2]) / _func[-1])
if self.criterion == "v":
crit = np.abs(
(_v_func[-1] - _v_func[-2]) / _v_func[-1])
_crit.append(crit)
if crit < self.eps:
_converged = True
# Store informations for this component
self.info.append(_info)
self.subfunc.append(_subfunc)
self.func.append(_func)
self.v_func.append(_v_func)
self.crit.append(_crit)
self.converged.append(_converged)
d_new = self.compute_d(X, u_new, v_new)
X = X - self.compute_rank1_approx(d_new, u_new, v_new)
self.U[:, j] = u_new[:, 0]
self.d[j] = d_new
self.V[:, j] = v_new[:, 0]
return self
def run(self, g, x):
"""Find the best separating margin for the samples in X.
Parameters
----------
g : MajoriserFunction
The function that majorises self.function.
x : array_like
The point at which to start the minimisation process.
"""
x = check_arrays(x)
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, False)
if self.info_requested(utils.Info.time):
_t = []
if self.info_requested(utils.Info.func_val):
_f = []
if self.info_requested(utils.Info.converged):
self.info_set(utils.Info.converged, False)
xnew = xold = x
for i in range(1, self.max_iter + 1):
if self.info_requested(utils.Info.time):
tm = utils.time()
xold = xnew
if isinstance(g, properties.MajoriserFunction):
maj_f = g(self.function, xold)
else:
g.at_point(xold)
maj_f = g
xnew = self.algorithm.run(maj_f, xold)
if self.info_requested(utils.Info.time):
_t.append(utils.time() - tm)
if self.info_requested(utils.Info.func_val):
_f.append(g.f(xnew))
if self.callback is not None:
self.callback(locals())
if i >= self.min_iter and np.linalg.norm(xnew - xold) < self.eps:
if self.info_requested(utils.Info.converged):
self.info_set(utils.Info.converged, True)
break
self.num_iter = i
if self.info_requested(utils.Info.num_iter):
self.info_set(utils.Info.num_iter, self.num_iter)
if self.info_requested(utils.Info.time):
self.info_set(utils.Info.time, _t)
if self.info_requested(utils.Info.func_val):
self.info_set(utils.Info.func_val, _f)
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, True)
return xnew
def run(self, X, y, alpha=None):
"""Find the best separating margin for the samples in X.
Parameters
----------
X : ndarray
The matrix with samples to separate.
y : array_like
The class belongings for the samples in X. Values must be -1
or 1.
alpha : numpy array
A start vector for the lagrange multipliers. Default is to use a
zero vector.
"""
X, y = check_arrays(X, check_array_in(y, [-1, 1]))
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, False)
if self.info_requested(utils.Info.time):
_t = time()
if self.info_requested(utils.Info.func_val):
self._f = []
n, p = X.shape
# Set up error cache
self._E = np.zeros(n)
self._Evalid = np.zeros(n, dtype=np.bool)
# Bias (intercept/threshold)
self.bias = 0.0
# Lagrange multipliers
if alpha is None:
self.alpha = np.zeros((n, 1))
else:
self.alpha = alpha
numChanged = 0
examineAll = True
while numChanged > 0 or examineAll:
numChanged = 0
if examineAll:
for i in xrange(n):
numChanged += self._examineSample(i, X, y)
else:
for i in xrange(n):
if self.alpha[i, 0] > 0.0 and self.alpha[i, 0] < self.C:
numChanged += self._examineSample(i, X, y)
if examineAll:
examineAll = False
elif numChanged == 0:
examineAll = True
if self.info_requested(utils.Info.time):
self.info_set(utils.Info.time, time() - _t)
if self.info_requested(utils.Info.func_val):
self.info_set(utils.Info.func_val, self._f)
del self._f # Remove for future runs
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, True)
return self._compute_w(X, y)
def run(self, g, x):
"""Find the best separating margin for the samples in X.
Parameters
----------
g : MajoriserFunction
The function that majorises self.function.
x : array_like
The point at which to start the minimisation process.
"""
x = check_arrays(x)
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, False)
if self.info_requested(utils.Info.time):
_t = []
if self.info_requested(utils.Info.func_val):
_f = []
if self.info_requested(utils.Info.converged):
self.info_set(utils.Info.converged, False)
xnew = xold = x
for i in range(1, self.max_iter + 1):
if self.info_requested(utils.Info.time):
tm = utils.time()
xold = xnew
if isinstance(g, properties.MajoriserFunction):
maj_f = g(self.function, xold)
else:
g.at_point(xold)
maj_f = g
xnew = self.algorithm.run(maj_f, xold)
if self.info_requested(utils.Info.time):
_t.append(utils.time() - tm)
if self.info_requested(utils.Info.func_val):
_f.append(g.f(xnew))
if self.callback is not None:
self.callback(locals())
if i >= self.min_iter and np.linalg.norm(xnew - xold) < self.eps:
if self.info_requested(utils.Info.converged):
self.info_set(utils.Info.converged, True)
break
self.num_iter = i
if self.info_requested(utils.Info.num_iter):
self.info_set(utils.Info.num_iter, self.num_iter)
if self.info_requested(utils.Info.time):
self.info_set(utils.Info.time, _t)
if self.info_requested(utils.Info.func_val):
self.info_set(utils.Info.func_val, _f)
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, True)
return xnew
