def add_batch(self, X, T, wc=None):
"""Add a batch of training data to an iterative solution, weighted if neeed.
The batch is processed as a whole, the training data is splitted in `ELM.add_data()` method.
With parameters HH_out, HT_out, the output will be put into these matrices instead of model.
Args:
X (matrix): input data matrix size (N * `inputs`)
T (matrix): output data matrix size (N * `outputs`)
wc (vector): vector of weights for data samples, one weight per sample, size (N * 1)
HH_out, HT_out (matrix, optional): output matrices to add batch result into, always given together
"""
devH = self._project(X, dev=True)
T = np.array(T, order="C", dtype=self.precision)
devT = gpuarray.to_gpu(T)
if wc is not None: # apply weights if given
w = np.array(
wc**0.5,
dtype=self.precision)[:, None] # re-shape to column matrix
devWC = gpuarray.to_gpu(w)
misc.mult_matvec(devH, devWC, axis=0, out=devH)
misc.mult_matvec(devT, devWC, axis=0, out=devT)
if self.HH is None: # initialize space for self.HH, self.HT
self.HT = misc.zeros((self.L, self.outputs), dtype=self.precision)
self.HH = linalg.eye(self.L, dtype=self.precision)
self.HH *= self.norm
linalg.add_dot(devH, devT, self.HT, transa='T')
if self.precision is np.float64:
linalg.add_dot(devH, devH, self.HH, transa='T')
else:
cublas.cublasSsyrk(self.handle, 'L', 'N', self.L, X.shape[0], 1,
devH.ptr, self.L, 1, self.HH.ptr, self.L)
def add_batch(self, X, T, wc=None):
"""Add a batch of training data to an iterative solution, weighted if neeed.
The batch is processed as a whole, the training data is splitted in `ELM.add_data()` method.
With parameters HH_out, HT_out, the output will be put into these matrices instead of model.
Args:
X (matrix): input data matrix size (N * `inputs`)
T (matrix): output data matrix size (N * `outputs`)
wc (vector): vector of weights for data samples, one weight per sample, size (N * 1)
HH_out, HT_out (matrix, optional): output matrices to add batch result into, always given together
"""
devH = self._project(X, dev=True)
T = np.array(T, order="C", dtype=self.precision)
devT = gpuarray.to_gpu(T)
if wc is not None: # apply weights if given
w = np.array(wc**0.5, dtype=self.precision)[:, None] # re-shape to column matrix
devWC = gpuarray.to_gpu(w)
misc.mult_matvec(devH, devWC, axis=0, out=devH)
misc.mult_matvec(devT, devWC, axis=0, out=devT)
if self.HH is None: # initialize space for self.HH, self.HT
self.HT = misc.zeros((self.L, self.outputs), dtype=self.precision)
self.HH = linalg.eye(self.L, dtype=self.precision)
self.HH *= self.norm
linalg.add_dot(devH, devT, self.HT, transa='T')
if self.precision is np.float64:
linalg.add_dot(devH, devH, self.HH, transa='T')
else:
cublas.cublasSsyrk(self.handle, 'L', 'N', self.L, X.shape[0], 1, devH.ptr, self.L, 1, self.HH.ptr, self.L)
def compute_analysis_cuda2(self,
xb,
y,
R,
P,
H,
HT=None,
hph=None,
calcP=True):
if HT is None:
HT = culinalg.transpose(H)
HP = culinalg.dot(H, P)
if hph is None:
hph = culinalg.dot(HP, HT)
Rhph = misc.add(R, hph)
inv = culinalg.inv(Rhph)
W = culinalg.dot(HP, inv, transa='T')
Hxb = culinalg.dot(H, xb)
yHxb = misc.subtract(y, Hxb)
WyHxb = culinalg.dot(W, yHxb)
xhat = misc.add(xb, WyHxb)
#xhat = xb + culinalg.dot(W, (y - culinalg.dot(H, xb)))
if calcP:
I = culinalg.eye(P.shape[0])
WH = culinalg.dot(W, H)
IWH = I - WH
Phat = culinalg.dot(IWH, P)
else:
Phat = misc.zeros((1, ), dtype=P.dtype)
return xhat, Phat
def test_eye_large_float32(self):
N = 128
e_gpu = linalg.eye(N, dtype=np.float32)
assert np.all(np.eye(N, dtype=np.float32) == e_gpu.get())
def test_eye_complex128(self):
N = 10
e_gpu = linalg.eye(N, dtype=np.complex128)
assert np.all(np.eye(N, dtype=np.complex128) == e_gpu.get())
def test_eye_float64(self):
N = 10
e_gpu = linalg.eye(N, dtype=np.float64)
assert np.all(np.eye(N, dtype=np.float64) == e_gpu.get())
def test_eye_large_float32(self):
N = 128
e_gpu = linalg.eye(N, dtype=np.float32)
assert np.all(np.eye(N, dtype=np.float32) == e_gpu.get())
def test_eye_complex128(self):
N = 10
e_gpu = linalg.eye(N, dtype=np.complex128)
assert np.all(np.eye(N, dtype=np.complex128) == e_gpu.get())
def test_eye_float64(self):
N = 10
e_gpu = linalg.eye(N, dtype=np.float64)
assert np.all(np.eye(N, dtype=np.float64) == e_gpu.get())
def almLasso_mat_fun(self):
'''
This function represents the Augumented Lagrangian Multipliers method for Lasso problem.
The lagrangian form of the Lasso can be expressed as following:
MIN{ 1/2||Y-XBHETA||_2^2 + lambda||THETA||_1} s.t B-T=0
When applied to this problem, the ADMM updates take the form
BHETA^t+1 = (XtX + rhoI)^-1(Xty + rho^t - mu^t)
THETA^t+1 = Shrinkage_lambda/rho(BHETA(t+1) + mu(t)/rho)
mu(t+1) = mu(t) + rho(BHETA(t+1) - BHETA(t+1))
The algorithm involves a 'ridge regression' update for BHETA, a soft-thresholding (shrinkage) step for THETA and
then a simple linear update for mu
NB: Actually, this ADMM version contains several variations such as the using of two penalty parameters instead
of just one of them (mu1, mu2)
'''
print('\tADMM processing...')
alpha1 = alpha2 = 0
if (len(self.reg_params) == 1):
alpha1 = self.reg_params[0]
alpha2 = self.reg_params[0]
elif (len(self.reg_params) == 2):
alpha1 = self.reg_params[0]
alpha2 = self.reg_params[1]
#thresholds parameters for stopping criteria
if (len(self.thr) == 1):
thr1 = self.thr[0]
thr2 = self.thr[0]
elif (len(self.thr) == 2):
thr1 = self.thr[0]
thr2 = self.thr[1]
# entry condition
err1 = 10 * thr1
err2 = 10 * thr2
start_time = time.time()
# setting penalty parameters for the ALM
mu1p = alpha1 * 1 / self.computeLambda()
print("\t\t-Compute Lambda- Time = %s seconds" %
(time.time() - start_time))
mu2p = alpha2 * 1
mu1 = mu1p
mu2 = mu2p
i = 1
start_time = time.time()
if self.GPU == True:
# defining penalty parameters e constraint to minimize, lambda and C matrix respectively
THETA = misc.zeros((self.num_columns, self.num_columns),
dtype='float64')
lambda2 = misc.zeros((self.num_columns, self.num_columns),
dtype='float64')
gpu_data = gpuarray.to_gpu(self.data)
P_GPU = linalg.dot(gpu_data, gpu_data, transa='T')
OP1 = P_GPU
linalg.scale(np.float32(mu1), OP1)
OP2 = linalg.eye(self.num_columns)
linalg.scale(mu2, OP2)
if self.affine == True:
print('\t\tGPU affine...')
OP3 = misc.ones((self.num_columns, self.num_columns),
dtype='float64')
linalg.scale(mu2, OP3)
lambda3 = misc.zeros((1, self.num_columns), dtype='float64')
# TODO: Because of some problem with linalg.inv version of scikit-cuda we fix it using np.linalg.inv of numpy
A = np.linalg.inv(
misc.add(misc.add(OP1.get(), OP2.get()), OP3.get()))
A_GPU = gpuarray.to_gpu(A)
while ((err1 > thr1 or err2 > thr1) and i < self.max_iter):
_lambda2 = gpuarray.to_gpu(lambda2)
_lambda3 = gpuarray.to_gpu(lambda3)
linalg.scale(1 / mu2, _lambda2)
term_OP2 = gpuarray.to_gpu(_lambda2.get())
OP2 = gpuarray.to_gpu(misc.subtract(THETA, term_OP2))
linalg.scale(mu2, OP2)
OP4 = gpuarray.to_gpu(
np.matlib.repmat(_lambda3.get(), self.num_columns, 1))
# updating Z
BHETA = linalg.dot(
A_GPU, misc.add(misc.add(misc.add(OP1, OP2), OP3),
OP4))
# deallocating unnecessary GPU variables
OP2.gpudata.free()
OP4.gpudata.free()
_lambda2.gpudata.free()
_lambda3.gpudata.free()
# updating C
THETA = misc.add(BHETA, term_OP2)
THETA = self.shrinkL1Lq(THETA.get(), 1 / mu2)
THETA = THETA.astype('float64')
# updating Lagrange multipliers
term_lambda2 = misc.subtract(BHETA, gpuarray.to_gpu(THETA))
linalg.scale(mu2, term_lambda2)
term_lambda2 = gpuarray.to_gpu(term_lambda2.get())
lambda2 = misc.add(lambda2, term_lambda2) # on GPU
term_lambda3 = misc.subtract(
misc.ones((1, self.num_columns), dtype='float64'),
misc.sum(BHETA, axis=0))
linalg.scale(mu2, term_lambda3)
term_lambda3 = gpuarray.to_gpu(term_lambda3.get())
lambda3 = misc.add(lambda3, term_lambda3) # on GPU
# deallocating unnecessary GPU variables
term_OP2.gpudata.free()
term_lambda2.gpudata.free()
term_lambda3.gpudata.free()
err1 = self.errorCoef(BHETA.get(), THETA)
err2 = self.errorCoef(np.sum(BHETA.get(), axis=0),
np.ones([1, self.num_columns]))
# deallocating unnecessary GPU variables
BHETA.gpudata.free()
THETA = gpuarray.to_gpu((THETA))
# reporting errors
if (self.verbose and (i % self.step == 0)):
print(
'\t\tIteration = %d, ||Z - C|| = %2.5e, ||1 - C^T 1|| = %2.5e'
% (i, err1, err2))
i += 1
THETA = THETA.get()
Err = [err1, err2]
if (self.verbose):
print(
'\t\tTerminating ADMM at iteration %5.0f, \n ||Z - C|| = %2.5e, ||1 - C^T 1|| = %2.5e. \n'
% (i, err1, err2))
else:
print '\t\tGPU not affine'
# TODO: Because of some problem with linalg.inv version of scikit-cuda we fix it using np.linalg.inv of numpy
A = np.linalg.inv(misc.add(OP1.get(), OP2.get()))
A_GPU = gpuarray.to_gpu(A)
while (err1 > thr1 and i < self.max_iter):
_lambda2 = gpuarray.to_gpu(lambda2)
term_OP2 = THETA
linalg.scale(mu2, term_OP2)
term_OP2 = misc.subtract(term_OP2, _lambda2)
OP2 = gpuarray.to_gpu(term_OP2.get())
BHETA = linalg.dot(A_GPU, misc.add(OP1, OP2))
linalg.scale(1 / mu2, _lambda2)
term_THETA = gpuarray.to_gpu(_lambda2.get())
THETA = misc.add(BHETA, term_THETA)
THETA = self.shrinkL1Lq(THETA.get(), 1 / mu2)
THETA = THETA.astype('float32')
# updating Lagrange multipliers
term_lambda2 = misc.subtract(BHETA, gpuarray.to_gpu(THETA))
linalg.scale(mu2, term_lambda2)
term_lambda2 = gpuarray.to_gpu(term_lambda2.get())
lambda2 = misc.add(lambda2, term_lambda2) # on GPU
err1 = self.errorCoef(BHETA.get(), THETA)
THETA = gpuarray.to_gpu((THETA))
# reporting errors
if (self.verbose and (i % self.step == 0)):
print('\t\tIteration %5.0f, ||Z - C|| = %2.5e' %
(i, err1))
i += 1
THETA = THETA.get()
Err = [err1, err2]
if (self.verbose):
print(
'\t\tTerminating ADMM at iteration %5.0f, \n ||Z - C|| = %2.5e'
% (i, err1))
else: #CPU version
# defining penalty parameters e constraint to minimize, lambda and C matrix respectively
THETA = np.zeros([self.num_columns, self.num_columns])
lambda2 = np.zeros([self.num_columns, self.num_columns])
P = self.data.T.dot(self.data)
OP1 = np.multiply(P, mu1)
if self.affine == True:
# INITIALIZATION
lambda3 = np.zeros(self.num_columns).T
A = np.linalg.inv(
np.multiply(mu1, P) +
np.multiply(mu2, np.eye(self.num_columns, dtype=int)) +
np.multiply(mu2,
np.ones([self.num_columns, self.num_columns])))
OP3 = np.multiply(
mu2, np.ones([self.num_columns, self.num_columns]))
while ((err1 > thr1 or err2 > thr1) and i < self.max_iter):
# updating Bheta
OP2 = np.multiply(THETA - np.divide(lambda2, mu2), mu2)
OP4 = np.matlib.repmat(lambda3, self.num_columns, 1)
BHETA = A.dot(OP1 + OP2 + OP3 + OP4)
# updating C
THETA = BHETA + np.divide(lambda2, mu2)
THETA = self.shrinkL1Lq(THETA, 1 / mu2)
# updating Lagrange multipliers
lambda2 = lambda2 + np.multiply(mu2, BHETA - THETA)
lambda3 = lambda3 + np.multiply(
mu2,
np.ones([1, self.num_columns]) - np.sum(BHETA, axis=0))
err1 = self.errorCoef(BHETA, THETA)
err2 = self.errorCoef(np.sum(BHETA, axis=0),
np.ones([1, self.num_columns]))
# mu1 = min(mu1 * (1 + 10 ^ -5), 10 ^ 2 * mu1p);
# mu2 = min(mu2 * (1 + 10 ^ -5), 10 ^ 2 * mu2p);
# reporting errors
if (self.verbose and (i % self.step == 0)):
print(
'\t\tIteration = %d, ||Z - C|| = %2.5e, ||1 - C^T 1|| = %2.5e'
% (i, err1, err2))
i += 1
Err = [err1, err2]
if (self.verbose):
print(
'\t\tTerminating ADMM at iteration %5.0f, \n ||Z - C|| = %2.5e, ||1 - C^T 1|| = %2.5e. \n'
% (i, err1, err2))
else:
print '\t\tCPU not affine'
A = np.linalg.inv(
OP1 +
np.multiply(mu2, np.eye(self.num_columns, dtype=int)))
while (err1 > thr1 and i < self.max_iter):
# updating Z
OP2 = np.multiply(mu2, THETA) - lambda2
BHETA = A.dot(OP1 + OP2)
# updating C
THETA = BHETA + np.divide(lambda2, mu2)
THETA = self.shrinkL1Lq(THETA, 1 / mu2)
# updating Lagrange multipliers
lambda2 = lambda2 + np.multiply(mu2, BHETA - THETA)
# computing errors
err1 = self.errorCoef(BHETA, THETA)
# reporting errors
if (self.verbose and (i % self.step == 0)):
print('\t\tIteration %5.0f, ||Z - C|| = %2.5e' %
(i, err1))
i += 1
Err = [err1, err2]
if (self.verbose):
print(
'\t\tTerminating ADMM at iteration %5.0f, \n ||Z - C|| = %2.5e'
% (i, err1))
print("\t\t-ADMM- Time = %s seconds" % (time.time() - start_time))
return THETA, Err
