def test_determinant_normalizer():
mat = np.random.randn(100, 100)
mat = np.dot(mat.T, mat)
det = np.linalg.det(mat)
det_norm = det**(1.0 / 100.0)
ndet = np.linalg.det(mat / det_norm)
pdet = np.linalg.det(mat / tikutils.determinant_normalizer(mat))
assert np.allclose(tikutils.determinant_normalizer(mat), det_norm)
assert np.allclose(ndet, 1.0)
assert np.allclose(pdet, 1.0)
def prior2penalty(self, dodetnorm=False):
penalty = np.linalg.inv(self.prior) # must be invertible
if self.penalty is None:
# first time
self.penalty = penalty
detnorm = tikutils.determinant_normalizer(penalty) if dodetnorm else 1.0
return penalty / detnorm
def test_determinant_normalizer():
mat = np.random.randn(100, 100)
mat = np.dot(mat.T, mat)
evals = np.linalg.eigvalsh(mat)
det = np.prod(evals)
det = det**(1. / len(evals))
assert np.allclose(tikutils.determinant_normalizer(mat), det)
def get_prior(self, alpha=1.0, wishart_lambda=0.0, dodetnorm=False):
'''
'''
# compute prior
prior = np.linalg.inv(self.penalty + wishart_lambda * self.wishart)
if dodetnorm:
prior /= tikutils.determinant_normalizer(prior)
return alpha**-2.0 * prior
def get_prior(self, alpha=1.0, hhparam=0.0, dodetnorm=False):
'''
'''
# compute prior
prior = np.linalg.inv(self.penalty + hhparam * self.wishart)
# select requested delays from prior
prior, delays = get_delays_from_prior(prior, self.delays)
if dodetnorm:
prior /= tikutils.determinant_normalizer(prior)
return alpha**-2.0 * prior
def __init__(self, penalty, delays=None, wishart=True, **kwargs):
'''
Parameters
----------
penalty : 2D np.ndarray (d, d)
Temporal penalty covariance. If the Tikhhov temporal penalty is C,
then the temporal penalty covariance is: np.dot(C.T, C)
delays : list_like, optional
Number of delays to use. Only positive delays supported.
Defaults to `np.arange(d)`
wishart : bool, optional
Regularize the temporal penalty covariance before taking
the inverse to compute the prior.
hhparams : list_like, optional
Parameters used to regularize the computation of the prior
from the penalty.
dodetnorm : bool, optional
Set the determinat of the initial prior to 1.
'''
prior = np.zeros_like(penalty)
if delays is None:
delays = np.arange(penalty.shape[0])
dodetnorm = False
if 'dodetnorm' in kwargs:
dodetnorm = kwargs.pop('dodetnorm')
super(PriorFromPenalty, self).__init__(prior, delays=delays, **kwargs)
# overwrite penalty after init
self.penalty = penalty.copy()
# set prior on prior
if isinstance(wishart, np.ndarray):
assert np.allclose(wishart.shape, self.penalty.shape)
self.wishart = wishart
elif wishart is True:
self.wishart = np.eye(self.penalty.shape[0])
elif wishart is False:
self.wishart = np.zeros_like(self.penalty)
else:
raise ValueError('invalid prior for prior')
self.dodetnorm = dodetnorm
# compute prior
prior = self.get_prior(alpha=1.0)
if self.dodetnorm:
prior /= tikutils.determinant_normalizer(prior)
self.prior = prior
def prior2penalty(self, regularizer=0.0, dodetnorm=True):
if regularizer > 0.0:
# make sure we have a valid prior
assert not np.allclose(self.prior, 0)
penalty = super(PriorFromPenalty,
self).prior2penalty(regularizer=regularizer,
dodetnorm=dodetnorm)
elif dodetnorm:
# re-scale
penalty = self.penalty / tikutils.determinant_normalizer(
self.penalty)
else:
# exact
penalty = self.penalty
return penalty
def test_base_prior():
tmp = np.random.randn(10, 10)
raw_prior = np.dot(tmp, tmp.T)
prior = tp.BasePrior(raw_prior, dodetnorm=False)
# basic defaults
assert prior.detnorm == 1.0
assert np.allclose(prior.asarray, raw_prior)
# return penalty
penalty = np.linalg.inv(raw_prior)
assert np.allclose(prior.prior2penalty(dodetnorm=False), penalty)
#####
prior = tp.BasePrior(raw_prior, dodetnorm=True)
# test determinant normalizer computation
detnorm = tikutils.determinant_normalizer(raw_prior)
assert prior.detnorm == detnorm
# test determinant normalization
rr = raw_prior / tikutils.determinant_normalizer(raw_prior)
assert np.allclose(prior.asarray, rr)
#####
prior = tp.BasePrior(raw_prior, dodetnorm=False)
# test penalty computation
penalty = np.linalg.inv(raw_prior)
assert np.allclose(prior.prior2penalty(dodetnorm=False), penalty)
assert prior.penalty_detnorm == 1.0
# test penalty with normalization
penalty_detnorm = prior.prior2penalty(dodetnorm=True)
assert prior.penalty_detnorm == 1.0
# update class
prior.normalize_penalty()
pdetnorm = tikutils.determinant_normalizer(penalty)
assert prior.penalty_detnorm == pdetnorm
assert np.allclose(penalty_detnorm, penalty / pdetnorm)
def get_prior(self, alpha=1.0, hhparam=1.0, dodetnorm=False):
'''
'''
if hhparam == self.kernel_object.kernel_parameter:
# it's already set, do nothing
pass
else:
# update gaussian width
self.kernel_object.update(hhparam)
prior = self.kernel_object.kernel
# select requested delays from prior
prior, delays = get_delays_from_prior(prior, self.delays)
if dodetnorm:
prior /= tikutils.determinant_normalizer(prior)
return alpha**-2.0 * prior
def get_prior(self, alpha=1.0, dodetnorm=False, **kwargs):
r'''
Parameters
----------
alpha : scalar
Regularization parameter (lambda).
Returns
-------
regularized_covar : 2D np.ndarray (p, p)
Scaled ridge matrix (i.e. :math:`\lambda^{-2} \Sigma_p`)
'''
if dodetnorm:
prior = self.asarray / tikutils.determinant_normalizer(self.asarray)
else:
prior = self.asarray
return alpha**-2.0 * prior
def get_prior(self, alpha=1.0, hhparam=0.0, dodetnorm=False):
r'''Convert the penalty to a prior.
Parameters
----------
alpha : scalar
The regularization scale on the prior (i.e. lambda)
hhparam : scalar
The regularization parameter for the hyper-prior.
Defaults to `0` for no regularization.
dodetnorm : bool, optional
Set the determinant of the MVN prior to 1.
Returns
-------
regularized_prior : 2D np.ndarray (d, d)
Regularized prior from penalty.
By default, no regularization is applied.
Notes
-----
.. math::
\Sigma = (P + \gamma I)^{-1}
where P is the temporal penalty covariance, :math:`\gamma` is the
hyper-prior parameter (`hhparam`).
'''
# compute prior
prior = np.linalg.inv(self.penalty + hhparam * self.wishart)
# select requested delays from prior
prior, delays = get_delays_from_prior(prior, self.delays)
if dodetnorm:
prior /= tikutils.determinant_normalizer(prior)
return alpha**-2.0 * prior
def get_prior(self, alpha=1.0, hhparam=1.0, dodetnorm=False):
'''Gaussian/RBF kernel as a temporal prior covariance matrix.
Parameters
----------
alpha : scalar, optional
The regularization scale (i.e. lambda).
hhparam : scalar, optional
The width of the Gaussian
Defaults to `1`.
dodetnorm : bool, optional
Set the determinat of the prior covariance matrix to 1.
'''
self.kernel_object.update(hhparam)
prior = self.kernel_object.kernel
# select requested delays from prior
prior, delays = get_delays_from_prior(prior, self.delays)
if dodetnorm:
prior /= tikutils.determinant_normalizer(prior)
return alpha**-2.0 * prior
def test_cv_api(show_figures=False, ntest=50):
# if show_figures=True, this function will create
# images of the temporal priors, and the feature prior hyparams in 3D
ridges = [0., 1e-03, 1., 10.0, 100.]
nridges = len(ridges)
ndelays = 10
delays = range(ndelays)
features_train, features_test, responses_train, responses_test = get_abc_data(
)
features_sizes = [fs.shape[1] for fs in features_train]
spatial_priors = [
sps.SphericalPrior(features_sizes[0]),
sps.SphericalPrior(features_sizes[1], hyparams=np.logspace(-3, 3, 7)),
sps.SphericalPrior(features_sizes[2], hyparams=np.logspace(-3, 3, 7)),
]
# do not scale first. this removes duplicates
spatial_priors[0].set_hyparams(1.0)
# non-diagonal hyper-prior
W = np.random.randn(ndelays, ndelays)
W = np.dot(W.T, W)
tpriors = [
tps.SphericalPrior(delays),
tps.SmoothnessPrior(delays, hhparams=np.logspace(-3, 1, 8)),
tps.SmoothnessPrior(delays, wishart=True),
tps.SmoothnessPrior(delays, wishart=False),
tps.SmoothnessPrior(delays, wishart=W, hhparams=np.logspace(-3, 3, 5)),
tps.GaussianKernelPrior(delays,
hhparams=np.linspace(1, ndelays / 2, ndelays)),
tps.HRFPrior([1] if delays == [0] else delays),
]
nfolds = (1, 5) # 1 times 5-fold cross-validation
folds = tikutils.generate_trnval_folds(responses_train.shape[0],
sampler='bcv',
nfolds=nfolds)
nfolds = np.prod(nfolds)
for ntp, temporal_prior in enumerate(tpriors):
print(temporal_prior)
all_temporal_hypers = [temporal_prior.get_hhparams()]
all_spatial_hypers = [t.get_hyparams() for t in spatial_priors]
# get all combinations of hyparams
all_hyperparams = list(
itertools.product(*(all_temporal_hypers + all_spatial_hypers)))
nspatial_hyperparams = np.prod([len(t) for t in all_spatial_hypers])
ntemporal_hyperparams = np.prod([len(t) for t in all_temporal_hypers])
population_mean = False
results = np.zeros(
(nfolds, ntemporal_hyperparams, nspatial_hyperparams, nridges,
1 if population_mean else responses_train.shape[-1]),
dtype=[
('fold', np.float32),
('tp', np.float32),
('sp', np.float32),
('ridges', np.float32),
('responses', np.float32),
])
for hyperidx, spatiotemporal_hyperparams in enumerate(all_hyperparams):
temporal_hyperparam = spatiotemporal_hyperparams[0]
spatial_hyperparams = spatiotemporal_hyperparams[1:]
spatial_hyperparams /= np.linalg.norm(spatial_hyperparams)
# get indices
shyperidx = np.mod(hyperidx, nspatial_hyperparams)
thyperidx = int(hyperidx // nspatial_hyperparams)
print(thyperidx,
temporal_hyperparam), (shyperidx, spatial_hyperparams)
this_temporal_prior = temporal_prior.get_prior(
alpha=1.0, hhparam=temporal_hyperparam)
if show_figures:
from matplotlib import pyplot as plt
if (hyperidx == 0) and (ntp == 0):
# show points in 3D
from tikreg import priors
cartesian_points = [
t[1:] / np.linalg.norm(t[1:]) for t in all_hyperparams
]
angles = priors.cartesian2polar(
np.asarray(cartesian_points))
priors.show_spherical_angles(angles[:, 0], angles[:, 1])
if hyperidx == 0:
# show priors with different hyper-priors
oldthyper = 0
plt.matshow(this_temporal_prior, cmap='inferno')
else:
if thyperidx > oldthyper:
oldthyper = thyperidx
plt.matshow(this_temporal_prior, cmap='inferno')
# only run a few
if hyperidx > ntest:
continue
Ktrain = 0.
Kval = 0.
for fdx, (fs_train, fs_test, fs_prior, fs_hyper) in enumerate(
zip(features_train, features_test, spatial_priors,
spatial_hyperparams)):
kernel_train = models.kernel_spatiotemporal_prior(
fs_train,
this_temporal_prior,
fs_prior.get_prior(fs_hyper),
delays=delays)
Ktrain += kernel_train
kernel_normalizer = tikutils.determinant_normalizer(Ktrain)
Ktrain /= float(kernel_normalizer)
# cross-validation
for ifold, (trnidx, validx) in enumerate(folds):
ktrn = tikutils.fast_indexing(Ktrain, trnidx, trnidx)
kval = tikutils.fast_indexing(Ktrain, validx, trnidx)
fit = models.solve_l2_dual(ktrn,
responses_train[trnidx],
kval,
responses_train[validx],
ridges=ridges,
verbose=False,
performance=True)
if population_mean:
cvfold = np.nan_to_num(fit['performance']).mean(-1)[...,
None]
else:
cvfold = fit['performance']
results[ifold, thyperidx, shyperidx] = cvfold
def normalize_prior(self):
self.detnorm = tikutils.determinant_normalizer(self.prior)
self.prior /= self.detnorm
def test_prior_from_penalty():
tmp = np.random.randn(10, 10)
raw_penalty = np.dot(tmp, tmp.T)
prior = tp.PriorFromPenalty(raw_penalty, dodetnorm=False)
assert np.allclose(raw_penalty, prior.penalty)
penalty = prior.prior2penalty(dodetnorm=False)
assert np.allclose(raw_penalty, penalty)
penalty = prior.prior2penalty(dodetnorm=True)
pdetnorm = tikutils.determinant_normalizer(raw_penalty)
assert np.allclose(raw_penalty / pdetnorm, penalty)
# this must break because the prior hasn't been updated
try:
penalty = prior.prior2penalty()
except AssertionError:
pass
# generate the prior from penalty
prior.get_prior(dodetnorm=False)
raw_prior = np.linalg.inv(raw_penalty)
assert np.allclose(raw_prior, prior.asarray)
assert np.allclose(prior.penalty, raw_penalty)
# generate a regularized prior from penalty
penalty = prior.prior2penalty(dodetnorm=True)
reg_penalty = np.linalg.inv(raw_prior)
reg_pdetnorm = tikutils.determinant_normalizer(reg_penalty)
assert np.allclose(prior.penalty, raw_penalty)
assert np.allclose(reg_penalty / reg_pdetnorm, penalty)
# subselect delays
delays = np.asarray([1, 2, 3, 4])
prior = tp.PriorFromPenalty(raw_penalty, delays=delays)
assert np.allclose(prior.delays, delays)
# generate prior
prior.get_prior(dodetnorm=False)
raw_prior = np.linalg.inv(raw_penalty)
raw_prior = tikutils.fast_indexing(raw_prior, delays, delays)
# prior should only contain delays of interest
assert np.allclose(raw_prior, prior.asarray)
# penalty should be kept as original
assert np.allclose(prior.penalty, raw_penalty)
# check default wishart covariance
assert np.allclose(prior.wishart, np.eye(raw_penalty.shape[0]))
# regularize penalty before inverting
oprior = prior.get_prior(hhparam=2.0, dodetnorm=False)
raw_prior = np.linalg.inv(raw_penalty + 2.0 * np.eye(raw_penalty.shape[0]))
raw_prior = tikutils.fast_indexing(raw_prior, delays, delays)
assert np.allclose(raw_prior, oprior)
# regularize
oprior = prior.get_prior(hhparam=2.0, dodetnorm=True)
detnorm = tikutils.determinant_normalizer(raw_prior)
assert np.allclose(raw_prior / detnorm, oprior)
# set a non-diagonal wishart prior
a = np.random.randn(10, 10)
W = np.dot(a, a.T)
prior.set_wishart(W)
assert np.allclose(prior.wishart, W)
# check the update works
oprior = prior.get_prior(hhparam=2.0, dodetnorm=True)
raw_prior = np.linalg.inv(raw_penalty + 2.0 * W)
raw_prior = tikutils.fast_indexing(raw_prior, delays, delays)
detnorm = tikutils.determinant_normalizer(raw_prior)
assert np.allclose(raw_prior / detnorm, oprior)
def get_prior(self, alpha=1.0, dodetnorm=False, **kwargs):
if dodetnorm:
prior = self.asarray / tikutils.determinant_normalizer(self.asarray)
else:
prior = self.asarray
return alpha**-2.0 * prior
def normalize_penalty(self):
self.penalty_detnorm = tikutils.determinant_normalizer(self.penalty)
self.penalty /= self.penalty_detnorm