def average_label_entropy(self,
X,
y,
theta=None,
eval_gradient=False,
verbose=False):
'''Evaluate the average label entropy of the Gaussian field model on a
dataset.
Parameters
----------
X: 2D array or list of objects
Feature vectors or other generic representations of input data.
y: 1D array
Label of each data point. Values of None or NaN indicates
missing labels that will be filled in by the model.
theta: 1D array
Hyperparameters for the weight class.
eval_gradients:
Whether or not to evaluate the gradient of the average label
entropy with respect to weight hyperparameters.
verbose: bool
If true, print out some additional information as a markdown table.
Returns
-------
average_label_entropy: float
The average label entropy of the Gaussian field prediction on the
unlabeled nodes.
grad: 1D array
Gradient with respect to the hyperparameters.
'''
if theta is not None:
self.weight.theta = theta
if eval_gradient is True:
z, dz, t_metric, t_solve, t_chain = self._predict_gradient(X, y)
else:
z = self._predict(X, y)
loss = -np.mean(z * np.log(z) + (1 - z) * np.log(1 - z))
if eval_gradient is True:
dloss = np.log(z) - np.log(1 - z)
grad = -np.mean(dloss * dz.T, axis=1) * np.exp(self.weight.theta)
retval = (loss, grad)
else:
retval = loss
if verbose and eval_gradient is True:
mprint.table(
('Avg.Entropy', '%12.5g', loss),
('Gradient', '%12.5g', np.linalg.norm(grad)),
('Metric time', '%12.2g', t_metric),
('Solver time', '%12.2g', t_solve),
('BackProp time', '%14.2g', t_chain),
)
return retval
def test_markdown_table_multicol():
out = sys.stdout = StringIO()
markdown.table(('Hello', '%9d', 0), ('Hello', '%12f', 0),
('Hello', '%15g', 0),
print_header=False)
cols = out.getvalue().strip().strip('|').split('|')
assert (len(cols) == 3)
assert (len(cols[0]) == 9)
assert (len(cols[1]) == 12)
assert (len(cols[2]) == 15)
def test_markdown_row_print_header():
out = sys.stdout = StringIO()
markdown.table_start()
markdown.table(
('Hello', '%9d', 0),
('Hello', '%12f', 0),
('Hello', '%15g', 0),
)
assert (len(out.getvalue().strip().split('\n')) == 3)
out = sys.stdout = StringIO()
markdown.table(
('Hello', '%9d', 0),
('Hello', '%12f', 0),
('Hello', '%15g', 0),
)
assert (len(out.getvalue().strip().split('\n')) == 1)
out = sys.stdout = StringIO()
markdown.table(('Hello', '%9d', 0), ('Hello', '%12f', 0),
('Hello', '%15g', 0),
print_header=True)
assert (len(out.getvalue().strip().split('\n')) == 3)
def log_marginal_likelihood(self,
theta=None,
C=None,
X=None,
y=None,
eval_gradient=False,
clone_kernel=True,
verbose=False):
"""Returns the log-marginal likelihood of a given set of log-scale
hyperparameters.
Parameters
----------
theta: array-like
Kernel hyperparameters for which the log-marginal likelihood is
to be evaluated. If None, the current hyperparameters will be used.
C: list of objects or feature vectors.
The core set that defines the subspace of low-rank approximation.
If None, `self.C` will be used.
X: list of objects or feature vectors.
Input values of the training data. If None, `self.X` will be used.
y: 1D array
Output/target values of the training data. If None, `self.y` will
be used.
eval_gradient: boolean
If True, the gradient of the log-marginal likelihood with respect
to the kernel hyperparameters at position theta will be returned
alongside.
clone_kernel: boolean
If True, the kernel is copied so that probing with theta does not
alter the trained kernel. If False, the kernel hyperparameters will
be modified in-place.
verbose: boolean
If True, the log-likelihood value and its components will be
printed to the screen.
Returns
-------
log_likelihood: float
Log-marginal likelihood of theta for training data.
log_likelihood_gradient: 1D array
Gradient of the log-marginal likelihood with respect to the kernel
hyperparameters at position theta. Only returned when eval_gradient
is True.
"""
theta = theta if theta is not None else self.kernel.theta
C = C if C is not None else self._C
X = X if X is not None else self._X
if y is not None:
y_mask, y = self.mask(y)
else:
y = self._y
y_mask = self._y_mask
if clone_kernel is True:
kernel = self.kernel.clone_with_theta(theta)
else:
kernel = self.kernel
kernel.theta = theta
t_kernel = time.perf_counter()
if eval_gradient is True:
Kxc, d_Kxc = self._gramian(None, X, C, kernel=kernel, jac=True)
Kcc, d_Kcc = self._gramian(self.alpha, C, kernel=kernel, jac=True)
Kxc, d_Kxc = Kxc[y_mask, :], d_Kxc[y_mask, :, :]
else:
Kxc = self._gramian(None, X, C, kernel=kernel)
Kcc = self._gramian(self.alpha, C, kernel=kernel)
Kxc = Kxc[y_mask, :]
t_kernel = time.perf_counter() - t_kernel
t_linalg = time.perf_counter()
Kcc_rsqrt = self._corespace(Kcc=Kcc)
F = Kxc @ Kcc_rsqrt
K = lr.dot(F, rcond=self.beta, mode='clamp')
K_inv = K.pinv()
logdet = K.logdet()
Ky = K_inv @ y
yKy = y @ Ky
logP = yKy + logdet
if eval_gradient is True:
D_theta = np.zeros_like(theta)
K_inv2 = K_inv**2
for i, t in enumerate(theta):
d_F = d_Kxc[:, :, i] @ Kcc_rsqrt
d_K = lr.dot(F, d_F.T) + lr.dot(d_F, F.T) - lr.dot(
F @ Kcc_rsqrt.T @ d_Kcc[:, :, i], Kcc_rsqrt @ F.T)
d_logdet = (K_inv @ d_K).trace()
d_Kinv_part = K_inv2 @ d_K - K_inv2 @ d_K @ (K @ K_inv)
d_Kinv = d_Kinv_part + d_Kinv_part.T - K_inv @ d_K @ K_inv
d_yKy = d_Kinv.quadratic(y, y)
D_theta[i] = (d_logdet + d_yKy) * np.exp(t)
retval = (logP, D_theta)
else:
retval = logP
t_linalg = time.perf_counter() - t_linalg
if verbose:
mprint.table(
('logP', '%12.5g', yKy + logdet),
('dlogP', '%12.5g', np.linalg.norm(D_theta)),
('y^T.K.y', '%12.5g', yKy),
('log|K| ', '%12.5g', logdet),
('Cond(K)', '%12.5g', K.cond()),
('GPU time', '%10.2g', t_kernel),
('CPU time', '%10.2g', t_linalg),
)
return retval
def squared_loocv_error(self,
theta=None,
X=None,
y=None,
eval_gradient=False,
clone_kernel=True,
verbose=False):
"""Returns the squared LOOCV error of a given set of log-scale
hyperparameters.
Parameters
----------
theta: array-like
Kernel hyperparameters for which the log-marginal likelihood is
to be evaluated. If None, the current hyperparameters will be used.
X: list of objects or feature vectors.
Input values of the training data. If None, `self.X` will be used.
y: 1D array
Output/target values of the training data. If None, `self.y` will
be used.
eval_gradient: boolean
If True, the gradient of the log-marginal likelihood with respect
to the kernel hyperparameters at position theta will be returned
alongside.
clone_kernel: boolean
If True, the kernel is copied so that probing with theta does not
alter the trained kernel. If False, the kernel hyperparameters will
be modified in-place.
verbose: boolean
If True, the log-likelihood value and its components will be
printed to the screen.
Returns
-------
squared_error: float
Squared LOOCV error of theta for training data.
squared_error_gradient: 1D array
Gradient of the Squared LOOCV error with respect to the kernel
hyperparameters at position theta. Only returned when eval_gradient
is True.
"""
theta = theta if theta is not None else self.kernel.theta
X = X if X is not None else self._X
if y is not None:
y_mask, y = self.mask(y)
else:
y = self._y
y_mask = self._y_mask
if clone_kernel is True:
kernel = self.kernel.clone_with_theta(theta)
else:
kernel = self.kernel
kernel.theta = theta
t_kernel = time.perf_counter()
if eval_gradient is True:
K, dK = self._gramian(self.alpha, X, kernel=kernel, jac=True)
K = K[y_mask, :][:, y_mask]
dK = dK[y_mask, :, :][:, y_mask, :]
else:
K = self._gramian(self.alpha, X, kernel=kernel)
K = K[y_mask, :][:, y_mask]
t_kernel = time.perf_counter() - t_kernel
t_linalg = time.perf_counter()
Kinv, logdet = self._invert(K, rcond=self.beta)
if not isinstance(Kinv, np.ndarray):
Kinv = Kinv.todense()
Kinv_diag = Kinv.diagonal()
Ky = Kinv @ y
e = Ky / Kinv_diag
squared_error = 0.5 * np.sum(e**2)
if eval_gradient is True:
D_theta = np.zeros_like(theta)
for i, t in enumerate(theta):
dk = dK[:, :, i]
KdK = Kinv @ dk
D_theta[i] = (
-(e / Kinv_diag) @ (KdK @ Ky) +
(e**2 / Kinv_diag) @ (KdK @ Kinv).diagonal()) * np.exp(t)
retval = (squared_error, D_theta)
else:
retval = squared_error
t_linalg = time.perf_counter() - t_linalg
if verbose:
mprint.table(
('Sq.Err.', '%12.5g', squared_error),
('d(SqErr)', '%12.5g', squared_error),
('log|K| ', '%12.5g', logdet),
('Cond(K)', '%12.5g', np.linalg.cond(K)),
('t_GPU (s)', '%10.2g', t_kernel),
('t_CPU (s)', '%10.2g', t_linalg),
)
return retval
def log_marginal_likelihood(self,
theta=None,
X=None,
y=None,
eval_gradient=False,
clone_kernel=True,
verbose=False):
"""Returns the log-marginal likelihood of a given set of log-scale
hyperparameters.
Parameters
----------
theta: array-like
Kernel hyperparameters for which the log-marginal likelihood is
to be evaluated. If None, the current hyperparameters will be used.
X: list of objects or feature vectors.
Input values of the training data. If None, `self.X` will be used.
y: 1D array
Output/target values of the training data. If None, `self.y` will
be used.
eval_gradient: boolean
If True, the gradient of the log-marginal likelihood with respect
to the kernel hyperparameters at position theta will be returned
alongside.
clone_kernel: boolean
If True, the kernel is copied so that probing with theta does not
alter the trained kernel. If False, the kernel hyperparameters will
be modified in-place.
verbose: boolean
If True, the log-likelihood value and its components will be
printed to the screen.
Returns
-------
log_likelihood: float
Log-marginal likelihood of theta for training data.
log_likelihood_gradient: 1D array
Gradient of the log-marginal likelihood with respect to the kernel
hyperparameters at position theta. Only returned when eval_gradient
is True.
"""
theta = theta if theta is not None else self.kernel.theta
X = X if X is not None else self._X
if y is not None:
y_mask, y = self.mask(y)
else:
y = self._y
y_mask = self._y_mask
if clone_kernel is True:
kernel = self.kernel.clone_with_theta(theta)
else:
kernel = self.kernel
kernel.theta = theta
t_kernel = time.perf_counter()
if eval_gradient is True:
K, dK = self._gramian(self.alpha, X, kernel=kernel, jac=True)
K = K[y_mask, :][:, y_mask]
dK = dK[y_mask, :, :][:, y_mask, :]
else:
K = self._gramian(self.alpha, X, kernel=kernel)
K = K[y_mask, :][:, y_mask]
t_kernel = time.perf_counter() - t_kernel
t_linalg = time.perf_counter()
Kinv, logdet = self._invert(K, rcond=self.beta)
Ky = Kinv @ y
yKy = y @ Ky
if eval_gradient is True:
if not isinstance(Kinv, np.ndarray):
Kinv = Kinv.todense()
d_theta = (np.einsum('ij,ijk->k', Kinv, dK) -
np.einsum('i,ijk,j', Ky, dK, Ky))
retval = (yKy + logdet, d_theta * np.exp(theta))
else:
retval = yKy + logdet
t_linalg = time.perf_counter() - t_linalg
if verbose:
mprint.table(
('logP', '%12.5g', yKy + logdet),
('dlogP', '%12.5g', np.linalg.norm(d_theta)),
('y^T.K.y', '%12.5g', yKy),
('log|K| ', '%12.5g', logdet),
('Cond(K)', '%12.5g', np.linalg.cond(K)),
('GPU time', '%10.2g', t_kernel),
('CPU time', '%10.2g', t_linalg),
)
return retval
def loocv_error(self,
X,
y,
p=2,
theta=None,
eval_gradient=False,
verbose=False):
'''Evaluate the leave-one-out cross validation error and gradient.
Parameters
----------
X: 2D array or list of objects
Feature vectors or other generic representations of input data.
y: 1D array
Label of each data point. Values of None or NaN indicates
missing labels that will be filled in by the model.
p: float > 1
The order of the p-norm for LOOCV error.
theta: 1D array
Hyperparameters for the weight class.
eval_gradients:
Whether or not to evaluate the gradient of the average label
entropy with respect to weight hyperparameters.
verbose: bool
If true, print out some additional information as a markdown table.
Returns
-------
err: 1D array
LOOCV Error
grad: 1D array
Gradient with respect to the hyperparameters.
'''
if theta is not None:
self.weight.theta = theta
labeled = np.isfinite(y)
y = y[labeled]
n = len(y)
t_metric = time.perf_counter()
if eval_gradient is True:
W, dW = self.weight(X[labeled], eval_gradient=True)
else:
if self.weight == 'precomputed':
W = X[labeled, :][:, labeled]
else:
W = self.weight(X[labeled])
t_metric = time.perf_counter() - t_metric
t_chain = time.perf_counter()
W += self.smoothing
D = W.sum(axis=1)
P = (1 / D)[:, None] * W
e = y - P @ y
loocv_error_p = np.mean(np.abs(e)**p)
loocv_error = loocv_error_p**(1 / p)
if eval_gradient is True:
derr_de = (loocv_error_p**(1 / p - 1) * np.abs(e)**(p - 1) *
np.sign(e) / n)
derr_dtheta = (np.einsum('pq, pqi',
(derr_de / D**2 * (W @ y))[:, None], dW) -
np.einsum('p, q, pqi', derr_de / D, y, dW))
retval = (loocv_error, derr_dtheta)
else:
retval = loocv_error
t_chain = time.perf_counter() - t_chain
if verbose and eval_gradient is True:
mprint.table(
('LOOCV Err.', '%12.5g', loocv_error),
('Gradient', '%12.5g', np.linalg.norm(derr_dtheta)),
('Metric time', '%12.2g', t_metric),
('BackProp time', '%14.2g', t_chain),
)
return retval
def log_marginal_likelihood(self,
theta_ext,
X=None,
y=None,
eval_gradient=False,
clone_kernel=True,
verbose=False):
"""Returns the log-marginal likelihood of a given set of log-scale
hyperparameters.
Parameters
----------
theta_ext: array-like
Kernel hyperparameters and per-sample noise prior for which the
log-marginal likelihood is to be evaluated. If None, the current
hyperparameters will be used.
X: list of objects or feature vectors.
Input values of the training data. If None, `self.X` will be used.
y: 1D array
Output/target values of the training data. If None, `self.y` will
be used.
eval_gradient: boolean
If True, the gradient of the log-marginal likelihood with respect
to the kernel hyperparameters at position theta will be returned
alongside.
clone_kernel: boolean
If True, the kernel is copied so that probing with theta does not
alter the trained kernel. If False, the kernel hyperparameters will
be modified in-place.
verbose: boolean
If True, the log-likelihood value and its components will be
printed to the screen.
Returns
-------
log_likelihood: float
Log-marginal likelihood of theta for training data.
log_likelihood_gradient: 1D array
Gradient of the log-marginal likelihood with respect to the kernel
hyperparameters at position theta. Only returned when eval_gradient
is True.
"""
X = X if X is not None else self._X
y = y if y is not None else self._y
theta, log_sigma = fold_like(theta_ext, (self.kernel.theta, y))
sigma = np.exp(log_sigma)
if clone_kernel is True:
kernel = self.kernel.clone_with_theta(theta)
else:
kernel = self.kernel
kernel.theta = theta
t_kernel = time.perf_counter()
if eval_gradient is True:
K, dK = self._gramian(sigma**2, X, kernel=kernel, jac=True)
else:
K = self._gramian(sigma**2, X, kernel=kernel)
t_kernel = time.perf_counter() - t_kernel
t_linalg = time.perf_counter()
Kinv, logdet = self._invert_pseudoinverse(K, rcond=self.beta)
Kinv_diag = Kinv.diagonal()
Ky = Kinv @ y
yKy = y @ Ky
if eval_gradient is True:
d_theta = (np.einsum('ij,ijk->k', Kinv, dK) -
np.einsum('i,ijk,j', Ky, dK, Ky))
d_alpha = (Kinv_diag - Ky**2) * 2 * sigma
retval = (yKy + logdet, np.concatenate(
(d_theta, d_alpha)) * np.exp(theta_ext))
else:
retval = yKy + logdet
t_linalg = time.perf_counter() - t_linalg
if verbose:
mprint.table(
('logP', '%12.5g', yKy + logdet),
('dlogP', '%12.5g', np.linalg.norm(d_theta)),
('y^T.K.y', '%12.5g', yKy),
('log|K| ', '%12.5g', logdet),
('Cond(K)', '%12.5g', np.linalg.cond(K)),
('GPU time', '%10.2g', t_kernel),
('CPU time', '%10.2g', t_linalg),
)
return retval