def test_determinant_normalizer(): mat = np.random.randn(100, 100) mat = np.dot(mat.T, mat) det = np.linalg.det(mat) det_norm = det**(1.0 / 100.0) ndet = np.linalg.det(mat / det_norm) pdet = np.linalg.det(mat / tikutils.determinant_normalizer(mat)) assert np.allclose(tikutils.determinant_normalizer(mat), det_norm) assert np.allclose(ndet, 1.0) assert np.allclose(pdet, 1.0)
def prior2penalty(self, dodetnorm=False): penalty = np.linalg.inv(self.prior) # must be invertible if self.penalty is None: # first time self.penalty = penalty detnorm = tikutils.determinant_normalizer(penalty) if dodetnorm else 1.0 return penalty / detnorm
def test_determinant_normalizer(): mat = np.random.randn(100, 100) mat = np.dot(mat.T, mat) evals = np.linalg.eigvalsh(mat) det = np.prod(evals) det = det**(1. / len(evals)) assert np.allclose(tikutils.determinant_normalizer(mat), det)
def get_prior(self, alpha=1.0, wishart_lambda=0.0, dodetnorm=False): ''' ''' # compute prior prior = np.linalg.inv(self.penalty + wishart_lambda * self.wishart) if dodetnorm: prior /= tikutils.determinant_normalizer(prior) return alpha**-2.0 * prior
def get_prior(self, alpha=1.0, hhparam=0.0, dodetnorm=False): ''' ''' # compute prior prior = np.linalg.inv(self.penalty + hhparam * self.wishart) # select requested delays from prior prior, delays = get_delays_from_prior(prior, self.delays) if dodetnorm: prior /= tikutils.determinant_normalizer(prior) return alpha**-2.0 * prior
def __init__(self, penalty, delays=None, wishart=True, **kwargs): ''' Parameters ---------- penalty : 2D np.ndarray (d, d) Temporal penalty covariance. If the Tikhhov temporal penalty is C, then the temporal penalty covariance is: np.dot(C.T, C) delays : list_like, optional Number of delays to use. Only positive delays supported. Defaults to `np.arange(d)` wishart : bool, optional Regularize the temporal penalty covariance before taking the inverse to compute the prior. hhparams : list_like, optional Parameters used to regularize the computation of the prior from the penalty. dodetnorm : bool, optional Set the determinat of the initial prior to 1. ''' prior = np.zeros_like(penalty) if delays is None: delays = np.arange(penalty.shape[0]) dodetnorm = False if 'dodetnorm' in kwargs: dodetnorm = kwargs.pop('dodetnorm') super(PriorFromPenalty, self).__init__(prior, delays=delays, **kwargs) # overwrite penalty after init self.penalty = penalty.copy() # set prior on prior if isinstance(wishart, np.ndarray): assert np.allclose(wishart.shape, self.penalty.shape) self.wishart = wishart elif wishart is True: self.wishart = np.eye(self.penalty.shape[0]) elif wishart is False: self.wishart = np.zeros_like(self.penalty) else: raise ValueError('invalid prior for prior') self.dodetnorm = dodetnorm # compute prior prior = self.get_prior(alpha=1.0) if self.dodetnorm: prior /= tikutils.determinant_normalizer(prior) self.prior = prior
def prior2penalty(self, regularizer=0.0, dodetnorm=True): if regularizer > 0.0: # make sure we have a valid prior assert not np.allclose(self.prior, 0) penalty = super(PriorFromPenalty, self).prior2penalty(regularizer=regularizer, dodetnorm=dodetnorm) elif dodetnorm: # re-scale penalty = self.penalty / tikutils.determinant_normalizer( self.penalty) else: # exact penalty = self.penalty return penalty
def test_base_prior(): tmp = np.random.randn(10, 10) raw_prior = np.dot(tmp, tmp.T) prior = tp.BasePrior(raw_prior, dodetnorm=False) # basic defaults assert prior.detnorm == 1.0 assert np.allclose(prior.asarray, raw_prior) # return penalty penalty = np.linalg.inv(raw_prior) assert np.allclose(prior.prior2penalty(dodetnorm=False), penalty) ##### prior = tp.BasePrior(raw_prior, dodetnorm=True) # test determinant normalizer computation detnorm = tikutils.determinant_normalizer(raw_prior) assert prior.detnorm == detnorm # test determinant normalization rr = raw_prior / tikutils.determinant_normalizer(raw_prior) assert np.allclose(prior.asarray, rr) ##### prior = tp.BasePrior(raw_prior, dodetnorm=False) # test penalty computation penalty = np.linalg.inv(raw_prior) assert np.allclose(prior.prior2penalty(dodetnorm=False), penalty) assert prior.penalty_detnorm == 1.0 # test penalty with normalization penalty_detnorm = prior.prior2penalty(dodetnorm=True) assert prior.penalty_detnorm == 1.0 # update class prior.normalize_penalty() pdetnorm = tikutils.determinant_normalizer(penalty) assert prior.penalty_detnorm == pdetnorm assert np.allclose(penalty_detnorm, penalty / pdetnorm)
def get_prior(self, alpha=1.0, hhparam=1.0, dodetnorm=False): ''' ''' if hhparam == self.kernel_object.kernel_parameter: # it's already set, do nothing pass else: # update gaussian width self.kernel_object.update(hhparam) prior = self.kernel_object.kernel # select requested delays from prior prior, delays = get_delays_from_prior(prior, self.delays) if dodetnorm: prior /= tikutils.determinant_normalizer(prior) return alpha**-2.0 * prior
def get_prior(self, alpha=1.0, dodetnorm=False, **kwargs): r''' Parameters ---------- alpha : scalar Regularization parameter (lambda). Returns ------- regularized_covar : 2D np.ndarray (p, p) Scaled ridge matrix (i.e. :math:`\lambda^{-2} \Sigma_p`) ''' if dodetnorm: prior = self.asarray / tikutils.determinant_normalizer(self.asarray) else: prior = self.asarray return alpha**-2.0 * prior
def get_prior(self, alpha=1.0, hhparam=0.0, dodetnorm=False): r'''Convert the penalty to a prior. Parameters ---------- alpha : scalar The regularization scale on the prior (i.e. lambda) hhparam : scalar The regularization parameter for the hyper-prior. Defaults to `0` for no regularization. dodetnorm : bool, optional Set the determinant of the MVN prior to 1. Returns ------- regularized_prior : 2D np.ndarray (d, d) Regularized prior from penalty. By default, no regularization is applied. Notes ----- .. math:: \Sigma = (P + \gamma I)^{-1} where P is the temporal penalty covariance, :math:`\gamma` is the hyper-prior parameter (`hhparam`). ''' # compute prior prior = np.linalg.inv(self.penalty + hhparam * self.wishart) # select requested delays from prior prior, delays = get_delays_from_prior(prior, self.delays) if dodetnorm: prior /= tikutils.determinant_normalizer(prior) return alpha**-2.0 * prior
def get_prior(self, alpha=1.0, hhparam=1.0, dodetnorm=False): '''Gaussian/RBF kernel as a temporal prior covariance matrix. Parameters ---------- alpha : scalar, optional The regularization scale (i.e. lambda). hhparam : scalar, optional The width of the Gaussian Defaults to `1`. dodetnorm : bool, optional Set the determinat of the prior covariance matrix to 1. ''' self.kernel_object.update(hhparam) prior = self.kernel_object.kernel # select requested delays from prior prior, delays = get_delays_from_prior(prior, self.delays) if dodetnorm: prior /= tikutils.determinant_normalizer(prior) return alpha**-2.0 * prior
def test_cv_api(show_figures=False, ntest=50): # if show_figures=True, this function will create # images of the temporal priors, and the feature prior hyparams in 3D ridges = [0., 1e-03, 1., 10.0, 100.] nridges = len(ridges) ndelays = 10 delays = range(ndelays) features_train, features_test, responses_train, responses_test = get_abc_data( ) features_sizes = [fs.shape[1] for fs in features_train] spatial_priors = [ sps.SphericalPrior(features_sizes[0]), sps.SphericalPrior(features_sizes[1], hyparams=np.logspace(-3, 3, 7)), sps.SphericalPrior(features_sizes[2], hyparams=np.logspace(-3, 3, 7)), ] # do not scale first. this removes duplicates spatial_priors[0].set_hyparams(1.0) # non-diagonal hyper-prior W = np.random.randn(ndelays, ndelays) W = np.dot(W.T, W) tpriors = [ tps.SphericalPrior(delays), tps.SmoothnessPrior(delays, hhparams=np.logspace(-3, 1, 8)), tps.SmoothnessPrior(delays, wishart=True), tps.SmoothnessPrior(delays, wishart=False), tps.SmoothnessPrior(delays, wishart=W, hhparams=np.logspace(-3, 3, 5)), tps.GaussianKernelPrior(delays, hhparams=np.linspace(1, ndelays / 2, ndelays)), tps.HRFPrior([1] if delays == [0] else delays), ] nfolds = (1, 5) # 1 times 5-fold cross-validation folds = tikutils.generate_trnval_folds(responses_train.shape[0], sampler='bcv', nfolds=nfolds) nfolds = np.prod(nfolds) for ntp, temporal_prior in enumerate(tpriors): print(temporal_prior) all_temporal_hypers = [temporal_prior.get_hhparams()] all_spatial_hypers = [t.get_hyparams() for t in spatial_priors] # get all combinations of hyparams all_hyperparams = list( itertools.product(*(all_temporal_hypers + all_spatial_hypers))) nspatial_hyperparams = np.prod([len(t) for t in all_spatial_hypers]) ntemporal_hyperparams = np.prod([len(t) for t in all_temporal_hypers]) population_mean = False results = np.zeros( (nfolds, ntemporal_hyperparams, nspatial_hyperparams, nridges, 1 if population_mean else responses_train.shape[-1]), dtype=[ ('fold', np.float32), ('tp', np.float32), ('sp', np.float32), ('ridges', np.float32), ('responses', np.float32), ]) for hyperidx, spatiotemporal_hyperparams in enumerate(all_hyperparams): temporal_hyperparam = spatiotemporal_hyperparams[0] spatial_hyperparams = spatiotemporal_hyperparams[1:] spatial_hyperparams /= np.linalg.norm(spatial_hyperparams) # get indices shyperidx = np.mod(hyperidx, nspatial_hyperparams) thyperidx = int(hyperidx // nspatial_hyperparams) print(thyperidx, temporal_hyperparam), (shyperidx, spatial_hyperparams) this_temporal_prior = temporal_prior.get_prior( alpha=1.0, hhparam=temporal_hyperparam) if show_figures: from matplotlib import pyplot as plt if (hyperidx == 0) and (ntp == 0): # show points in 3D from tikreg import priors cartesian_points = [ t[1:] / np.linalg.norm(t[1:]) for t in all_hyperparams ] angles = priors.cartesian2polar( np.asarray(cartesian_points)) priors.show_spherical_angles(angles[:, 0], angles[:, 1]) if hyperidx == 0: # show priors with different hyper-priors oldthyper = 0 plt.matshow(this_temporal_prior, cmap='inferno') else: if thyperidx > oldthyper: oldthyper = thyperidx plt.matshow(this_temporal_prior, cmap='inferno') # only run a few if hyperidx > ntest: continue Ktrain = 0. Kval = 0. for fdx, (fs_train, fs_test, fs_prior, fs_hyper) in enumerate( zip(features_train, features_test, spatial_priors, spatial_hyperparams)): kernel_train = models.kernel_spatiotemporal_prior( fs_train, this_temporal_prior, fs_prior.get_prior(fs_hyper), delays=delays) Ktrain += kernel_train kernel_normalizer = tikutils.determinant_normalizer(Ktrain) Ktrain /= float(kernel_normalizer) # cross-validation for ifold, (trnidx, validx) in enumerate(folds): ktrn = tikutils.fast_indexing(Ktrain, trnidx, trnidx) kval = tikutils.fast_indexing(Ktrain, validx, trnidx) fit = models.solve_l2_dual(ktrn, responses_train[trnidx], kval, responses_train[validx], ridges=ridges, verbose=False, performance=True) if population_mean: cvfold = np.nan_to_num(fit['performance']).mean(-1)[..., None] else: cvfold = fit['performance'] results[ifold, thyperidx, shyperidx] = cvfold
def normalize_prior(self): self.detnorm = tikutils.determinant_normalizer(self.prior) self.prior /= self.detnorm
def test_prior_from_penalty(): tmp = np.random.randn(10, 10) raw_penalty = np.dot(tmp, tmp.T) prior = tp.PriorFromPenalty(raw_penalty, dodetnorm=False) assert np.allclose(raw_penalty, prior.penalty) penalty = prior.prior2penalty(dodetnorm=False) assert np.allclose(raw_penalty, penalty) penalty = prior.prior2penalty(dodetnorm=True) pdetnorm = tikutils.determinant_normalizer(raw_penalty) assert np.allclose(raw_penalty / pdetnorm, penalty) # this must break because the prior hasn't been updated try: penalty = prior.prior2penalty() except AssertionError: pass # generate the prior from penalty prior.get_prior(dodetnorm=False) raw_prior = np.linalg.inv(raw_penalty) assert np.allclose(raw_prior, prior.asarray) assert np.allclose(prior.penalty, raw_penalty) # generate a regularized prior from penalty penalty = prior.prior2penalty(dodetnorm=True) reg_penalty = np.linalg.inv(raw_prior) reg_pdetnorm = tikutils.determinant_normalizer(reg_penalty) assert np.allclose(prior.penalty, raw_penalty) assert np.allclose(reg_penalty / reg_pdetnorm, penalty) # subselect delays delays = np.asarray([1, 2, 3, 4]) prior = tp.PriorFromPenalty(raw_penalty, delays=delays) assert np.allclose(prior.delays, delays) # generate prior prior.get_prior(dodetnorm=False) raw_prior = np.linalg.inv(raw_penalty) raw_prior = tikutils.fast_indexing(raw_prior, delays, delays) # prior should only contain delays of interest assert np.allclose(raw_prior, prior.asarray) # penalty should be kept as original assert np.allclose(prior.penalty, raw_penalty) # check default wishart covariance assert np.allclose(prior.wishart, np.eye(raw_penalty.shape[0])) # regularize penalty before inverting oprior = prior.get_prior(hhparam=2.0, dodetnorm=False) raw_prior = np.linalg.inv(raw_penalty + 2.0 * np.eye(raw_penalty.shape[0])) raw_prior = tikutils.fast_indexing(raw_prior, delays, delays) assert np.allclose(raw_prior, oprior) # regularize oprior = prior.get_prior(hhparam=2.0, dodetnorm=True) detnorm = tikutils.determinant_normalizer(raw_prior) assert np.allclose(raw_prior / detnorm, oprior) # set a non-diagonal wishart prior a = np.random.randn(10, 10) W = np.dot(a, a.T) prior.set_wishart(W) assert np.allclose(prior.wishart, W) # check the update works oprior = prior.get_prior(hhparam=2.0, dodetnorm=True) raw_prior = np.linalg.inv(raw_penalty + 2.0 * W) raw_prior = tikutils.fast_indexing(raw_prior, delays, delays) detnorm = tikutils.determinant_normalizer(raw_prior) assert np.allclose(raw_prior / detnorm, oprior)
def get_prior(self, alpha=1.0, dodetnorm=False, **kwargs): if dodetnorm: prior = self.asarray / tikutils.determinant_normalizer(self.asarray) else: prior = self.asarray return alpha**-2.0 * prior
def normalize_penalty(self): self.penalty_detnorm = tikutils.determinant_normalizer(self.penalty) self.penalty /= self.penalty_detnorm