def post_transform(self, z_list, train=False):
if train:
self.cca = MCCA(latent_dims=self.latent_dims)
z_list = self.cca.fit_transform(z_list)
else:
z_list = self.cca.transform(z_list)
return z_list
def test_unregularized_multi():
# Tests unregularized CCA methods for more than 2 views. The idea is that all of these should give the same result.
latent_dims = 2
cca = rCCA(latent_dims=latent_dims).fit((X, Y, Z))
iter = CCA_ALS(latent_dims=latent_dims,
stochastic=False,
tol=1e-12,
random_state=rng).fit((X, Y, Z))
gcca = GCCA(latent_dims=latent_dims).fit((X, Y, Z))
mcca = MCCA(latent_dims=latent_dims).fit((X, Y, Z))
kcca = KCCA(latent_dims=latent_dims).fit((X, Y, Z))
corr_cca = cca.score((X, Y, Z))
corr_iter = iter.score((X, Y, Z))
corr_gcca = gcca.score((X, Y, Z))
corr_mcca = mcca.score((X, Y, Z))
corr_kcca = kcca.score((X, Y, Z))
# Check the correlations from each unregularized method are the same
assert np.testing.assert_array_almost_equal(corr_cca, corr_iter,
decimal=1) is None
assert np.testing.assert_array_almost_equal(corr_cca, corr_mcca,
decimal=1) is None
assert np.testing.assert_array_almost_equal(corr_cca, corr_gcca,
decimal=1) is None
assert np.testing.assert_array_almost_equal(corr_cca, corr_kcca,
decimal=1) is None
def test_sparse_input():
# Tests unregularized CCA methods. The idea is that all of these should give the same result.
latent_dims = 2
cca = CCA(latent_dims=latent_dims, centre=False).fit((X_sp, Y_sp))
iter = CCA_ALS(
latent_dims=latent_dims,
tol=1e-9,
stochastic=False,
centre=False,
random_state=rng,
).fit((X_sp, Y_sp))
iter_pls = PLS_ALS(latent_dims=latent_dims, tol=1e-9, centre=False).fit(
(X_sp, Y_sp))
gcca = GCCA(latent_dims=latent_dims, centre=False).fit((X_sp, Y_sp))
mcca = MCCA(latent_dims=latent_dims, centre=False).fit((X_sp, Y_sp))
kcca = KCCA(latent_dims=latent_dims, centre=False).fit((X_sp, Y_sp))
scca = SCCA(latent_dims=latent_dims, centre=False, c=0.001).fit(
(X_sp, Y_sp))
corr_cca = cca.score((X, Y))
corr_iter = iter.score((X, Y))
corr_gcca = gcca.score((X, Y))
corr_mcca = mcca.score((X, Y))
corr_kcca = kcca.score((X, Y))
# Check the correlations from each unregularized method are the same
assert np.testing.assert_array_almost_equal(
corr_iter, corr_mcca, decimal=1) is None
assert np.testing.assert_array_almost_equal(
corr_iter, corr_gcca, decimal=1) is None
assert np.testing.assert_array_almost_equal(
corr_iter, corr_kcca, decimal=1) is None
def test_unregularized_methods():
# Tests unregularized CCA methods. The idea is that all of these should give the same result.
latent_dims = 2
cca = CCA(latent_dims=latent_dims).fit([X, Y])
iter = CCA_ALS(latent_dims=latent_dims,
tol=1e-9,
stochastic=False,
random_state=rng).fit([X, Y])
gcca = GCCA(latent_dims=latent_dims).fit([X, Y])
mcca = MCCA(latent_dims=latent_dims, eps=1e-9).fit([X, Y])
kcca = KCCA(latent_dims=latent_dims).fit([X, Y])
kgcca = KGCCA(latent_dims=latent_dims).fit([X, Y])
tcca = TCCA(latent_dims=latent_dims).fit([X, Y])
corr_cca = cca.score((X, Y))
corr_iter = iter.score((X, Y))
corr_gcca = gcca.score((X, Y))
corr_mcca = mcca.score((X, Y))
corr_kcca = kcca.score((X, Y))
corr_kgcca = kgcca.score((X, Y))
corr_tcca = tcca.score((X, Y))
assert np.testing.assert_array_almost_equal(corr_cca, corr_iter,
decimal=1) is None
assert np.testing.assert_array_almost_equal(corr_cca, corr_mcca,
decimal=1) is None
assert np.testing.assert_array_almost_equal(corr_cca, corr_gcca,
decimal=1) is None
assert np.testing.assert_array_almost_equal(corr_cca, corr_kcca,
decimal=1) is None
assert np.testing.assert_array_almost_equal(corr_cca, corr_tcca,
decimal=1) is None
assert (np.testing.assert_array_almost_equal(
corr_kgcca, corr_gcca, decimal=1) is None)
# Check standardized models have standard outputs
assert (np.testing.assert_allclose(
np.linalg.norm(iter.transform(
(X, Y))[0], axis=0)**2, n, rtol=0.2) is None)
assert (np.testing.assert_allclose(
np.linalg.norm(cca.transform(
(X, Y))[0], axis=0)**2, n, rtol=0.2) is None)
assert (np.testing.assert_allclose(
np.linalg.norm(mcca.transform(
(X, Y))[0], axis=0)**2, n, rtol=0.2) is None)
assert (np.testing.assert_allclose(
np.linalg.norm(kcca.transform(
(X, Y))[0], axis=0)**2, n, rtol=0.2) is None)
assert (np.testing.assert_allclose(
np.linalg.norm(iter.transform(
(X, Y))[1], axis=0)**2, n, rtol=0.2) is None)
assert (np.testing.assert_allclose(
np.linalg.norm(cca.transform(
(X, Y))[1], axis=0)**2, n, rtol=0.2) is None)
assert (np.testing.assert_allclose(
np.linalg.norm(mcca.transform(
(X, Y))[1], axis=0)**2, n, rtol=0.2) is None)
assert (np.testing.assert_allclose(
np.linalg.norm(kcca.transform(
(X, Y))[1], axis=0)**2, n, rtol=0.2) is None)
class DCCA(_DCCA_base):
"""
A class used to fit a DCCA model.
:Citation:
Andrew, Galen, et al. "Deep canonical correlation analysis." International conference on machine learning. PMLR, 2013.
"""
def __init__(
self,
latent_dims: int,
objective=objectives.MCCA,
encoders=None,
r: float = 0,
eps: float = 1e-5,
):
"""
Constructor class for DCCA
:param latent_dims: # latent dimensions
:param objective: # CCA objective: normal tracenorm CCA by default
:param encoders: list of encoder networks
:param r: regularisation parameter of tracenorm CCA like ridge CCA. Needs to be VERY SMALL. If you get errors make this smaller
:param eps: epsilon used throughout. Needs to be VERY SMALL. If you get errors make this smaller
"""
super().__init__(latent_dims=latent_dims)
if encoders is None:
encoders = [Encoder, Encoder]
self.encoders = torch.nn.ModuleList(encoders)
self.objective = objective(latent_dims, r=r, eps=eps)
def forward(self, *args):
z = []
for i, encoder in enumerate(self.encoders):
z.append(encoder(args[i]))
return z
def loss(self, *args):
"""
Define the loss function for the model. This is used by the DeepWrapper class
:param args:
:return:
"""
z = self(*args)
return {"objective": self.objective.loss(*z)}
def post_transform(self, z_list, train=False):
if train:
self.cca = MCCA(latent_dims=self.latent_dims)
z_list = self.cca.fit_transform(z_list)
else:
z_list = self.cca.transform(z_list)
return z_list
def test_cv_fit(self):
latent_dims = 5
c1 = [0.1, 0.2]
c2 = [0.1, 0.2]
param_candidates = {'c': list(itertools.product(c1, c2))}
wrap_unweighted_gcca = GCCA(latent_dims=latent_dims).gridsearch_fit(self.X, self.Y, folds=2,
param_candidates=param_candidates,
plot=True)
wrap_deweighted_gcca = GCCA(latent_dims=latent_dims, view_weights=[0.5, 0.5]).gridsearch_fit(
self.X, self.Y, folds=2, param_candidates=param_candidates)
wrap_mcca = MCCA(latent_dims=latent_dims).gridsearch_fit(
self.X, self.Y, folds=2, param_candidates=param_candidates)
def test_regularized_methods():
# Test that linear regularized methods match PLS solution when using maximum regularisation.
latent_dims = 2
c = 1
kernel = KCCA(latent_dims=latent_dims,
c=[c, c],
kernel=["linear", "linear"]).fit((X, Y))
pls = PLS(latent_dims=latent_dims).fit([X, Y])
gcca = GCCA(latent_dims=latent_dims, c=[c, c]).fit([X, Y])
mcca = MCCA(latent_dims=latent_dims, c=[c, c]).fit([X, Y])
rcca = rCCA(latent_dims=latent_dims, c=[c, c]).fit([X, Y])
corr_gcca = gcca.score((X, Y))
corr_mcca = mcca.score((X, Y))
corr_kernel = kernel.score((X, Y))
corr_pls = pls.score((X, Y))
corr_rcca = rcca.score((X, Y))
# Check the correlations from each unregularized method are the same
assert np.testing.assert_array_almost_equal(corr_pls, corr_mcca,
decimal=1) is None
assert (np.testing.assert_array_almost_equal(
corr_pls, corr_kernel, decimal=1) is None)
assert np.testing.assert_array_almost_equal(corr_pls, corr_rcca,
decimal=1) is None
def _default_initializer(views, initialization, random_state, latent_dims):
"""
This is a generator function which generates initializations for each dimension
:param views:
:param initialization:
:param random_state:
:param latent_dims:
:return:
"""
if initialization == "random":
while True:
yield np.array([
random_state.normal(0, 1, size=(view.shape[0]))
for view in views
])
elif initialization == "uniform":
while True:
yield np.array([np.ones(view.shape[0]) for view in views])
elif initialization == "pls":
latent_dim = 0
# use rCCA to use multiple views
pls_scores = MCCA(latent_dims, c=1).fit_transform(views)
while True:
yield np.stack(pls_scores)[:, :, latent_dim]
latent_dim += 1
elif initialization == "cca":
latent_dim = 0
cca_scores = MCCA(latent_dims).fit_transform(views)
while True:
yield np.stack(cca_scores)[:, :, latent_dim]
latent_dim += 1
else:
raise ValueError(
"Initialization {type} not supported. Pass a generator implementing this method"
)
def test_regularized_methods(self):
# Test that linear regularized methods match PLS solution when using maximum regularisation
latent_dims = 5
c = 1
wrap_kernel = KCCA(latent_dims=latent_dims, c=[c, c], kernel=['linear', 'linear']).fit(self.X,
self.Y)
wrap_pls = PLS(latent_dims=latent_dims).fit(self.X, self.Y)
wrap_gcca = GCCA(latent_dims=latent_dims, c=[c, c]).fit(self.X, self.Y)
wrap_mcca = MCCA(latent_dims=latent_dims, c=[c, c]).fit(self.X, self.Y)
wrap_rCCA = rCCA(latent_dims=latent_dims, c=[c, c]).fit(self.X, self.Y)
corr_gcca = wrap_gcca.train_correlations[0, 1]
corr_mcca = wrap_mcca.train_correlations[0, 1]
corr_kernel = wrap_kernel.train_correlations[0, 1]
corr_pls = wrap_pls.train_correlations[0, 1]
corr_rcca = wrap_rCCA.train_correlations[0, 1]
# Check the correlations from each unregularized method are the same
# self.assertIsNone(np.testing.assert_array_almost_equal(corr_pls, corr_gcca, decimal=2))
self.assertIsNone(np.testing.assert_array_almost_equal(corr_pls, corr_mcca, decimal=1))
self.assertIsNone(np.testing.assert_array_almost_equal(corr_pls, corr_kernel, decimal=1))
self.assertIsNone(np.testing.assert_array_almost_equal(corr_pls, corr_rcca, decimal=1))
def test_unregularized_methods(self):
latent_dims = 1
wrap_cca = CCA(latent_dims=latent_dims).fit(self.X, self.Y)
wrap_iter = CCA_ALS(latent_dims=latent_dims, tol=1e-9).fit(self.X, self.Y)
wrap_gcca = GCCA(latent_dims=latent_dims).fit(self.X, self.Y)
wrap_mcca = MCCA(latent_dims=latent_dims).fit(self.X, self.Y)
wrap_kcca = KCCA(latent_dims=latent_dims).fit(self.X, self.Y)
corr_cca = wrap_cca.train_correlations[0, 1]
corr_iter = wrap_iter.train_correlations[0, 1]
corr_gcca = wrap_gcca.train_correlations[0, 1]
corr_mcca = wrap_mcca.train_correlations[0, 1]
corr_kcca = wrap_kcca.train_correlations[0, 1]
# Check the score outputs are the right shape
self.assertTrue(wrap_iter.score_list[0].shape == (self.X.shape[0], latent_dims))
self.assertTrue(wrap_gcca.score_list[0].shape == (self.X.shape[0], latent_dims))
self.assertTrue(wrap_mcca.score_list[0].shape == (self.X.shape[0], latent_dims))
self.assertTrue(wrap_kcca.score_list[0].shape == (self.X.shape[0], latent_dims))
# Check the correlations from each unregularized method are the same
self.assertIsNone(np.testing.assert_array_almost_equal(corr_cca, corr_iter, decimal=2))
self.assertIsNone(np.testing.assert_array_almost_equal(corr_iter, corr_mcca, decimal=2))
self.assertIsNone(np.testing.assert_array_almost_equal(corr_iter, corr_gcca, decimal=2))
self.assertIsNone(np.testing.assert_array_almost_equal(corr_iter, corr_kcca, decimal=2))
class DCCA(_DCCA_base, torch.nn.Module):
"""
A class used to fit a DCCA model.
Examples
--------
>>> from cca_zoo.deepmodels import DCCA
>>> model = DCCA()
"""
def __init__(self,
latent_dims: int,
objective=objectives.CCA,
encoders: List[BaseEncoder] = [Encoder, Encoder],
learning_rate=1e-3,
r: float = 1e-3,
eps: float = 1e-9,
schedulers: List = None,
optimizers: List[torch.optim.Optimizer] = None):
"""
Constructor class for DCCA
:param latent_dims: # latent dimensions
:param objective: # CCA objective: normal tracenorm CCA by default
:param encoders: list of encoder networks
:param learning_rate: learning rate if no optimizers passed
:param r: regularisation parameter of tracenorm CCA like ridge CCA
:param eps: epsilon used throughout
:param schedulers: list of schedulers for each optimizer
:param optimizers: list of optimizers for each encoder
"""
super().__init__(latent_dims)
self.latent_dims = latent_dims
self.encoders = torch.nn.ModuleList(encoders)
self.objective = objective(latent_dims, r=r)
if optimizers is None:
self.optimizers = [
torch.optim.Adam(list(encoder.parameters()), lr=learning_rate)
for encoder in self.encoders
]
else:
self.optimizers = optimizers
self.schedulers = []
if schedulers:
self.schedulers.extend(schedulers)
self.covs = None
self.eps = eps
def update_weights(self, *args):
[optimizer.zero_grad() for optimizer in self.optimizers]
z = self(*args)
loss = self.objective.loss(*z)
loss.backward()
[optimizer.step() for optimizer in self.optimizers]
return loss
def forward(self, *args):
z = self.encode(*args)
return z
def encode(self, *args):
z = []
for i, encoder in enumerate(self.encoders):
z.append(encoder(args[i]))
return tuple(z)
def loss(self, *args):
z = self(*args)
return self.objective.loss(*z)
def post_transform(self, *z_list, train=False):
if train:
self.cca = MCCA(latent_dims=self.latent_dims)
self.cca.fit(*z_list)
z_list = self.cca.transform(*z_list)
else:
z_list = self.cca.transform(*z_list)
return z_list
p = 3
q = 3
r = 3
latent_dims = 1
cv = 3
(X, Y, Z), (tx, ty, tz) = generate_covariance_data(
n, view_features=[p, q, r], latent_dims=latent_dims, correlation=[0.9]
)
# %%
# Eigendecomposition-Based Methods
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# %%
mcca = MCCA(latent_dims=latent_dims).fit((X, Y, X)).score((X, Y, Z))
# %%
gcca = GCCA(latent_dims=latent_dims).fit((X, Y, X)).score((X, Y, Z))
# %%
# We can also use kernel versions of these methods
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# %%
kcca = KCCA(latent_dims=latent_dims).fit((X, Y, X)).score((X, Y, Z))
# %%
kgcca = KGCCA(latent_dims=latent_dims).fit((X, Y, X)).score((X, Y, Z))
# %%
def test_data_gen(self):
(x, y, z), true_feats = generate_covariance_data(1000, [10, 11, 12], 1, [0.5, 0.5, 0.5], correlation=0.5)
cca = CCA().fit(x[:500], y[:500])
cca_pred = cca.predict_corr(x[500:], y[500:])
mcca = MCCA().fit(x[:500], y[:500], z[:500])
mcca_pred = mcca.predict_corr(x[500:], y[500:], z[500:])
class DCCA_NOI(DCCA, torch.nn.Module):
"""
A class used to fit a DCCA model by non-linear orthogonal iterations
Examples
--------
>>> from cca_zoo.deepmodels import DCCA_NOI
>>> model = DCCA_NOI()
"""
def __init__(self,
latent_dims: int,
objective=objectives.MCCA,
encoders: List[BaseEncoder] = [Encoder, Encoder],
learning_rate=1e-3,
r: float = 1e-3,
rho: float = 0.2,
eps: float = 1e-9,
shared_target: bool = False,
schedulers: List = None,
optimizers: List[torch.optim.Optimizer] = None):
"""
Constructor class for DCCA
:param latent_dims: # latent dimensions
:param objective: # CCA objective: normal tracenorm CCA by default
:param encoders: list of encoder networks
:param learning_rate: learning rate if no optimizers passed
:param r: regularisation parameter of tracenorm CCA like ridge CCA
:param rho: covariance memory like DCCA non-linear orthogonal iterations paper
:param eps: epsilon used throughout
:param shared_target: not used
:param schedulers: list of schedulers for each optimizer
:param optimizers: list of optimizers for each encoder
"""
super().__init__(latent_dims=latent_dims,
objective=objective,
encoders=encoders,
learning_rate=learning_rate,
r=r,
eps=eps,
schedulers=schedulers,
optimizers=optimizers)
self.latent_dims = latent_dims
self.encoders = torch.nn.ModuleList(encoders)
self.objective = objective(latent_dims, r=r)
if optimizers is None:
self.optimizers = [
torch.optim.Adam(list(encoder.parameters()), lr=learning_rate)
for encoder in self.encoders
]
else:
self.optimizers = optimizers
self.schedulers = []
if schedulers:
self.schedulers.extend(schedulers)
self.covs = None
self.eps = eps
self.rho = rho
self.shared_target = shared_target
assert (0 <= self.rho <= 1), "rho should be between 0 and 1"
def update_weights(self, *args):
z = self(*args)
self.update_covariances(*z)
covariance_inv = [
objectives._compute_matrix_power(cov, -0.5, self.eps)
for cov in self.covs
]
preds = [
torch.matmul(z, covariance_inv[i]).detach()
for i, z in enumerate(z)
]
losses = [
torch.mean(torch.norm(z_i - preds[-i], dim=0))
for i, z_i in enumerate(z, start=1)
]
obj = self.objective.loss(*z)
self.optimizers[0].zero_grad()
losses[0].backward()
self.optimizers[0].step()
self.optimizers[1].zero_grad()
losses[1].backward()
self.optimizers[1].step()
return obj
def forward(self, *args):
z = self.encode(*args)
return z
def encode(self, *args):
z = []
for i, encoder in enumerate(self.encoders):
z.append(encoder(args[i]))
return tuple(z)
def loss(self, *args):
z = self(*args)
return self.objective.loss(*z)
def update_covariances(self, *args):
b = args[0].shape[0]
batch_covs = [z_i.T @ z_i for i, z_i in enumerate(args)]
if self.covs is not None:
self.covs = [
(self.rho * self.covs[i]).detach() + (1 - self.rho) * batch_cov
for i, batch_cov in enumerate(batch_covs)
]
else:
self.covs = batch_covs
def post_transform(self, *z_list, train=False):
if train:
self.cca = MCCA(latent_dims=self.latent_dims)
self.cca.fit(*z_list)
z_list = self.cca.transform(*z_list)
else:
z_list = self.cca.transform(*z_list)
return z_list
"""
### (Regularized) Generalized CCA via alternating least squares (can pass more than 2 views)
"""
gcca = GCCA(latent_dims=latent_dims, c=[1, 1])
gcca.fit(train_view_1, train_view_2)
gcca_results = np.stack(
(gcca.train_correlations[0, 1], gcca.predict_corr(test_view_1,
test_view_2)[0, 1]))
"""
### (Regularized) Multiset CCA via alternating least squares (can pass more than 2 views)
"""
mcca = MCCA(latent_dims=latent_dims, c=[0.5, 0.5])
# small ammount of regularisation added since data is not full rank
mcca.fit(train_view_1, train_view_2)
mcca_results = np.stack(
(mcca.train_correlations[0, 1], mcca.predict_corr(test_view_1,
test_view_2)[0, 1]))
"""
### (Regularized) Tensor CCA via alternating least squares (can pass more than 2 views)
"""
tcca = TCCA(latent_dims=latent_dims, c=[0.99, 0.99])
# small ammount of regularisation added since data is not full rank
tcca.fit(train_view_1, train_view_2)