def lsmr_annihilate(x: csc_matrix, y: ndarray, use_cache: bool = True, x_hash=None,
**lsmr_options) -> ndarray:
r"""
Removes projection of x on y from y
Parameters
----------
x : csc_matrix
Sparse array of regressors
y : ndarray
Array with shape (nobs, nvar)
use_cache : bool
Flag indicating whether results should be stored in the cache,
and retrieved if available.
x_hash : object
Hashable object representing the values in x
lsmr_options: dict
Dictionary of options to pass to scipy.sparse.linalg.lsmr
Returns
-------
resids : ndarray
Returns the residuals from regressing y on x, (nobs, nvar)
Notes
-----
Residuals are estiamted column-by-column as
.. math::
\hat{\epsilon}_{j} = y_{j} - x^\prime \hat{\beta}
where :math:`\hat{\beta}` is computed using lsmr.
"""
use_cache = use_cache and x_hash is not None
regressor_hash = x_hash if x_hash is not None else ''
default_opts = dict(atol=1e-8, btol=1e-8, show=False)
default_opts.update(lsmr_options)
resids = []
for i in range(y.shape[1]):
_y = y[:, i:i + 1]
variable_digest = ''
if use_cache:
hasher = hash_func()
hasher.update(ascontiguousarray(_y.data))
variable_digest = hasher.hexdigest()
if use_cache and variable_digest in _VARIABLE_CACHE[regressor_hash]:
resid = _VARIABLE_CACHE[regressor_hash][variable_digest]
else:
beta = lsmr(x, _y, **default_opts)[0]
resid = y[:, i:i + 1] - (x.dot(csc_matrix(beta[:, None]))).A
_VARIABLE_CACHE[regressor_hash][variable_digest] = resid
resids.append(resid)
if resids:
return column_stack(resids)
else:
return empty_like(y)
def fit(self, X: csc_matrix, y):
print("Fit starts")
# X, X_, y, y_ = train_test_split(X, y, self.test_fraction)
e = self.predict(X) - y
q = X.dot(self.V)
n_samples, n_features = X.shape
X = X.tocsc()
for i in range(self.n_iter):
# self.evaluate(i, X_, y_)
# Global bias
w0_ = -(e - self.w0).sum() / (n_samples + self.lambda_w0)
e += w0_ - self.w0
self.w0 = w0_
# self.evaluate(i, X_, y_)
# 1-way interaction
for l in range(n_features):
Xl = X.getcol(l).toarray()
print(("\r Iteration #{} 1-way interaction "
"progress {:.2%}; train error {}").format(
i, l / n_features, err(e)),
end="")
w_ = -((e - self.w[l] * Xl) * Xl).sum() / (
np.power(Xl, 2).sum() + self.lambda_w)
e += (w_ - self.w[l]) * Xl
self.w[l] = w_
# self.evaluate(i, X_, y_)
# 2-way interaction
for f in range(self.latent_dimension):
Qf = q[:, f].reshape(-1, 1)
for l in range(n_features):
Xl = X.getcol(l)
idx = Xl.nonzero()[0]
Xl = Xl.data.reshape(-1, 1)
Vlf = self.V[l, f]
print((
"\r Iteration #{} 2-way interaction progress {:.2%};" +
"error {:.5}; validation_error NO").format(
i, (f * n_features + l) /
(self.latent_dimension * n_features), err(e)),
end="")
h = Xl * Qf[idx] - np.power(Xl, 2) * Vlf
v_ = -((e[idx] - Vlf * h) * h).sum() / (
np.power(h, 2).sum() + self.lambda_v)
e[idx] += (v_ - Vlf) * h
Qf[idx] += (v_ - Vlf) * Xl
self.V[l, f] = v_
q[:, f] = Qf.reshape(-1)
def trust_region(x: np.ndarray, g: np.ndarray, hess: np.ndarray,
scaling: csc_matrix, delta: float, dv: np.ndarray,
theta: float, lb: np.ndarray, ub: np.ndarray,
subspace_dim: SubSpaceDim,
stepback_strategy: StepBackStrategy, refine_stepback: bool,
logger: logging.Logger) -> Step:
"""
Compute a step according to the solution of the trust-region subproblem.
If step-back is necessary, gradient and reflected trust region step are
also evaluated in terms of their performance according to the local
quadratic approximation
:param x:
Current values of the optimization variables
:param g:
Objective function gradient at x
:param hess:
(Approximate) objective function Hessian at x
:param scaling:
Scaling transformation according to distance to boundary
:param delta:
Trust region radius, note that this applies after scaling
transformation
:param dv:
derivative of scaling transformation
:param theta:
parameter regulating stepback
:param lb:
lower optimization variable boundaries
:param ub:
upper optimization variable boundaries
:param subspace_dim:
Subspace dimension in which the subproblem will be solved. Larger
subspaces require more compute time but can yield higher quality step
proposals.
:param stepback_strategy:
Strategy that is applied when the proposed step exceeds the
optimization boundary.
:param refine_stepback:
If set to True, proposed steps that are computed via the specified
stepback_strategy will be refined via optimization.
:param logger:
logging.Logger instance to be used for logging
:return:
s: proposed step,
ss: rescaled proposed step,
qpval: expected function value according to local quadratic
approximation,
subspace: computed subspace for reuse if proposed step is not accepted,
steptype: type of step that was selected for proposal
"""
sg = scaling.dot(g)
g_dscaling = csc_matrix(np.diag(np.abs(g) * dv))
if subspace_dim == SubSpaceDim.TWO:
tr_step = TRStep2D(x, sg, hess, scaling, g_dscaling, delta, theta, ub,
lb, logger)
elif subspace_dim == SubSpaceDim.FULL:
tr_step = TRStepFull(x, sg, hess, scaling, g_dscaling, delta, theta,
ub, lb, logger)
else:
raise ValueError('Invalid choice of subspace dimension.')
tr_step.calculate()
# in case of truncation, we hit the boundary and we check both the
# gradient and the reflected step, either of which could be better than the
# TR step
steps = [tr_step]
if tr_step.alpha < 1.0 and len(g) > 1:
g_step = GradientStep(x, sg, hess, scaling, g_dscaling, delta, theta,
ub, lb, logger)
g_step.calculate()
steps.append(g_step)
if stepback_strategy == StepBackStrategy.SINGLE_REFLECT:
rtr_step = TRStepReflected(x, sg, hess, scaling, g_dscaling, delta,
theta, ub, lb, tr_step)
rtr_step.calculate()
steps.append(rtr_step)
if stepback_strategy in [
StepBackStrategy.REFLECT, StepBackStrategy.MIXED
]:
steps.extend(
stepback_reflect(tr_step, x, sg, hess, scaling, g_dscaling,
delta, theta, ub, lb))
if stepback_strategy in [
StepBackStrategy.TRUNCATE, StepBackStrategy.MIXED
]:
steps.extend(
stepback_truncate(tr_step, x, sg, hess, scaling, g_dscaling,
delta, theta, ub, lb))
if refine_stepback:
steps.extend(
stepback_refine(steps, x, sg, hess, scaling, g_dscaling, delta,
theta, ub, lb))
if len(steps) > 1:
rcountstrs = [
str(step.reflection_count) * int(step.reflection_count > 0)
for step in steps
]
logger.debug(' | '.join([
f'{step.type + rcountstr}: [qp:'
f' {step.qpval:.2E}, '
f'a: {step.alpha:.2E}]'
for rcountstr, step in zip(rcountstrs, steps)
]))
qpvals = [step.qpval for step in steps]
return steps[int(np.argmin(qpvals))]
def trust_region(x: np.ndarray, g: np.ndarray, hess: np.ndarray,
scaling: csc_matrix, delta: float, dv: np.ndarray,
theta: float, lb: np.ndarray, ub: np.ndarray,
subspace_dim: SubSpaceDim,
stepback_strategy: StepBackStrategy,
logger: logging.Logger) -> Step:
"""
Compute a step according to the solution of the trust-region subproblem.
If step-back is necessary, gradient and reflected trust region step are
also evaluated in terms of their performance according to the local
quadratic approximation
:param x:
Current values of the optimization variables
:param g:
Objective function gradient at x
:param hess:
(Approximate) objective function Hessian at x
:param scaling:
Scaling transformation according to distance to boundary
:param delta:
Trust region radius, note that this applies after scaling
transformation
:param dv:
derivative of scaling transformation
:param theta:
parameter regulating stepback
:param lb:
lower optimization variable boundaries
:param ub:
upper optimization variable boundaries
:param subspace_dim:
Subspace dimension in which the subproblem will be solved. Larger
subspaces require more compute time but can yield higher quality step
proposals.
:param stepback_strategy:
Strategy that is applied when the proposed step exceeds the
optimization boundary.
:param logger:
logging.Logger instance to be used for logging
:return:
s: proposed step,
"""
sg = scaling.dot(g)
# diag(g_k)*J^v_k Eq (2.5) [ColemanLi1994]
g_dscaling = csc_matrix(np.diag(np.abs(g) * dv))
step_options = {
SubSpaceDim.TWO: TRStep2D,
SubSpaceDim.FULL: TRStepFull,
SubSpaceDim.STEIHAUG: TRStepSteihaug,
}
tr_step = step_options[subspace_dim](x, sg, hess, scaling, g_dscaling,
delta, theta, ub, lb, logger)
tr_step.calculate()
# in case of truncation, we hit the boundary and we check both the
# gradient and the reflected step, either of which could be better than the
# TR step
steps = [tr_step]
if tr_step.alpha < 1.0 and len(g) > 1:
g_step = GradientStep(x, sg, hess, scaling, g_dscaling, delta, theta,
ub, lb, logger)
g_step.calculate()
steps.append(g_step)
if stepback_strategy == StepBackStrategy.SINGLE_REFLECT:
rtr_step = TRStepReflected(x, sg, hess, scaling, g_dscaling, delta,
theta, ub, lb, tr_step)
rtr_step.calculate()
steps.append(rtr_step)
if stepback_strategy in [
StepBackStrategy.REFLECT, StepBackStrategy.MIXED
]:
steps.extend(
stepback_reflect(tr_step, x, sg, hess, scaling, g_dscaling,
delta, theta, ub, lb))
if stepback_strategy in [
StepBackStrategy.TRUNCATE, StepBackStrategy.MIXED
]:
steps.extend(
stepback_truncate(tr_step, x, sg, hess, scaling, g_dscaling,
delta, theta, ub, lb))
if stepback_strategy == StepBackStrategy.REFINE and \
tr_step.subspace.shape[1] > 1:
ref_step = RefinedStep(x, sg, hess, scaling, g_dscaling, delta,
theta, ub, lb, tr_step)
ref_step.calculate()
steps.append(ref_step)
if len(steps) > 1:
rcountstrs = [
str(step.reflection_count) * int(step.reflection_count > 0)
for step in steps
]
logger.debug(' | '.join([
f'{step.type + rcountstr}: [qp:'
f' {step.qpval:.2E}, '
f'a: {step.alpha:.2E}]'
for rcountstr, step in zip(rcountstrs, steps)
]))
qpvals = [step.qpval for step in steps]
return steps[np.argmin(qpvals)]
def optimize(sparse_km: csc_matrix, gamma: float,
regcoef: float, L1: float, eps: float, max_iter: int) -> (csc_matrix, dict):
"""
Perform SVM on sparsified kernel matrix.
:param sparse_km: sparsified kernel matrix.
:param eps: epsilon value.
:param max_iter: maximal number of iterations.
:return: object weights.
"""
if sparse_km.shape[0] != sparse_km.shape[1]:
raise Exception("Kernel matrix is not a squared matrix")
log = {"grad_norm": [], "time": []}
N = sparse_km.shape[0]
def grad_f(x):
t = x.copy()
t.data -= 1 / (2 * N * regcoef)
return -csr_matrix((N, 1)) + sparse_km.dot(x) - \
gamma*sparse_clip(-x, 0, None) + gamma*sparse_clip(t, 0, None)
x0 = csr_matrix((N, 1))
x0[0, 0] = 1/2
grad_f0 = grad_f(x0)
grad_min = BasicGradientUpdater(grad_f0.T)
grad_max = BasicGradientUpdater(-grad_f0.T)
iter_counter = 0
start = timeit.default_timer()
current_point = x0
true_grad = grad_f0
while grad_min.get_norm() > eps**2 or iter_counter < max_iter:
#if true_grad == grad_min.get():
log["grad_norm"].append(grad_min.get_norm())
log["time"].append(timeit.default_timer() - start)
i_plus = grad_max.get_coordinate()
g_plus = -grad_max.get_value()
i_minus = grad_min.get_coordinate()
g_minus = grad_min.get_value()
h_val = 1/(4*L1)*(g_plus - g_minus)
h = csr_matrix((N, 1))
h[i_plus, 0] = h_val
h[i_minus, 0] = -h_val
t = current_point.copy() # had to make this turnaround 'cause "sparse vector + constant" operation hasn't been implemented yet
t.data -= 1 / (2 * N * regcoef)
delta_grad = sparse_km.dot(h)
delta_grad -= gamma*sparse_clip(-current_point - h, 0, None)
delta_grad += gamma*sparse_clip(-current_point, 0, None)
delta_grad += gamma*sparse_clip(t + h, 0, None)
delta_grad -= gamma*sparse_clip(t, 0, None)
grad_min.update(delta_grad.T)
grad_max.update(-delta_grad.T)
current_point += h
true_grad = grad_f(current_point.T)
iter_counter += 1
return h, log