def train(self, views, verbose=True):
tic = time.time()
assert len(np.shape(
views)) == 3, "Data shape must be 3D, i.e. sub x time x voxel"
self.Share = None
self.TrainFeatures = None
NumSub, NumTime, NumVoxel = np.shape(views)
NumFea = self.net_shape[-1]
if NumFea is None:
NumFea = np.min((NumTime, NumVoxel))
if verbose:
print(
"Number of features is automatically assigned, Features: ",
NumFea)
self.net_shape[-1] = NumFea
Share = np.random.randn(NumTime, NumFea)
if self.loss_type == 'mse':
criterion = torch.nn.MSELoss()
elif self.loss_type == 'soft':
criterion = torch.nn.MultiLabelSoftMarginLoss()
elif self.loss_type == 'mean':
criterion = torch.mean
elif self.loss_type == 'norm':
criterion = torch.norm
else:
raise Exception(
"Loss function type is wrong! Options: \'mse\', \'soft\', \'mean\', or \'norm\'"
)
self.ha_loss_vec = list()
self.ha_loss = None
for j in range(self.iteration):
NewViews = list()
G = torch.Tensor(Share)
for s in range(NumSub):
net_shape = np.concatenate(([NumVoxel], self.net_shape))
net = MLP(model=net_shape,
activation=self.activation,
gpu_enable=self.gpu_enable)
if self.optim == "adam":
optimizer = optim.Adam(net.parameters(),
lr=self.learning_rate)
elif self.optim == "sgd":
optimizer = optim.SGD(net.parameters(),
lr=self.learning_rate)
else:
raise Exception(
"Optimization algorithm is wrong! Options: \'adam\' or \'sgd\'"
)
X = torch.Tensor(views[s])
net.train()
for epoch in range(self.epoch):
perm = torch.randperm(NumTime)
sum_loss = 0
for i in range(0, NumTime, self.batch_size):
x = X[perm[i:i + self.batch_size]]
g = G[perm[i:i + self.batch_size]]
# Send data to GPU
if self.gpu_enable:
x = x.cuda()
g = g.cuda()
optimizer.zero_grad()
fx = net(x)
if self.loss_type == 'mse' or self.loss_type == 'soft':
loss = criterion(fx, g) / NumTime
else:
loss = criterion(fx - g) / NumTime
if self.norm1_enable or self.norm2_enable:
for weight in net.get_weights():
if self.norm1_enable:
loss += self.alpha * torch.mean(
torch.abs(weight[1]))
if self.norm2_enable:
loss += self.alpha * torch.mean(weight[1]**
2)
loss.backward()
optimizer.step()
sum_loss += loss.data.cpu().numpy()
if self.epoch_internal_iteration > (i / NumTime):
break
if verbose:
print(
"TRAIN, UPDATE NETWORK: Iteration {:5d}, Subject {:6d}, Epoch {:6d}, loss error: {}"
.format(j + 1, s + 1, epoch + 1, sum_loss))
if self.gpu_enable:
X = X.cuda()
NewViews.append(net(X).data.cpu().numpy())
ha_model = GPUHA(Dim=NumFea, regularization=self.regularization)
if NumFea >= NumTime:
ha_model.train(views=NewViews,
verbose=verbose,
gpu=self.gpu_enable)
else:
ha_model.train(views=NewViews, verbose=verbose, gpu=False)
Share = ha_model.G
out_features = ha_model.Xtrain
error = np.mean(ha_model.Etrain)
if error == 0:
assert self.Share is not None, "All extracted features are zero, i.e. number of features is not enough for creating a shared space"
self.TrainRuntime = time.time() - tic
return self.TrainFeatures, self.Share
if self.best_result_enable:
if self.ha_loss is None:
self.Share = Share
self.TrainFeatures = out_features
self.ha_loss = error
if error <= self.ha_loss:
self.Share = Share
self.TrainFeatures = out_features
self.ha_loss = error
else:
self.Share = Share
self.TrainFeatures = out_features
self.ha_loss = error
if verbose:
print("Hyperalignment error: {}".format(error))
self.ha_loss_vec.append(error)
self.TrainRuntime = time.time() - tic
return self.TrainFeatures, self.Share
def fit(self, data_vals, design_vals):
tic = time.time()
SampleSize, FeatureSize = np.shape(data_vals)
Sam, RegressorSize = np.shape(design_vals)
assert SampleSize == Sam, "Data and Design Matrix must have the same size samples, data shape: " + \
str(np.shape(data_vals)) + ", design shape: " + str(np.shape(design_vals))
del Sam
A = torch.Tensor(design_vals)
B = torch.Tensor(data_vals)
if self.normalization:
A = A - A.mean() / A.std()
B = B - B.mean() / B.std()
model = MLP([RegressorSize, FeatureSize], [None],
gpu_enable=self.gpu_enable)
if self.optim == "adam":
optimizer = optim.Adam(model.parameters(), lr=self.learning_rate)
elif self.optim == "sgd":
optimizer = optim.SGD(model.parameters(), lr=self.learning_rate)
else:
raise Exception(
"Optimization algorithm is wrong! Options: \'adam\' or \'sgd\'"
)
if self.loss_type == 'mse':
criterion = torch.nn.MSELoss()
elif self.loss_type == 'soft':
criterion = torch.nn.MultiLabelSoftMarginLoss()
elif self.loss_type == 'mean':
criterion = torch.mean
elif self.loss_type == 'norm':
criterion = torch.norm
else:
raise Exception(
"Loss function type is wrong! Options: \'mse\', \'soft\', \'mean\', or \'norm\'"
)
ModelIsWrong = True
for mod in [
'linear', 'lasso', 'elastic', 'ridge', 'ln1', 'ln2', 'ln12'
]:
if self.method == mod:
ModelIsWrong = False
break
if ModelIsWrong:
raise Exception(
"Method option is wrong! Options: \'linear\', \'lasso\', \'elastic\', \'ridge\', \'ln1\', \'ln2\', \'ln12\'"
)
model.train()
self.loss_vec = list()
for epoch in range(self.epoch):
perm = torch.randperm(SampleSize)
sum_loss = 0
for i in range(0, SampleSize, self.batch_size):
a = A[perm[i:i + self.batch_size]]
b = B[perm[i:i + self.batch_size]]
# Send data to GPU
if self.gpu_enable:
a = a.cuda()
b = b.cuda()
optimizer.zero_grad()
output = model(a)
W = model.get_weights()[0][1]
if self.method == 'linear':
# ||y - Xw||
if self.loss_type == 'mse' or self.loss_type == 'soft':
loss = criterion(output, b)
else:
loss = criterion(output - b)
elif self.method == 'lasso':
# (1 / (2 * n_samples)) * ||y - Xw|| + alpha * ||w||_1
if self.loss_type == 'mse' or self.loss_type == 'soft':
loss = criterion(output, b) / (
SampleSize * 2) + self.lasso_alpha * torch.norm(
W, p=1)
else:
loss = criterion(output - b) / (
SampleSize * 2) + self.lasso_alpha * torch.norm(
W, p=1)
elif self.method == 'elastic':
# 1 / (2 * n_samples) * ||y - Xw|| +
# alpha * l1_ratio * ||w||_1 +
# 0.5 * alpha * (1 - l1_ratio) * ||w||^2_2
if self.loss_type == 'mse' or self.loss_type == 'soft':
loss = criterion(output, b) / (SampleSize * 2) + \
self.elstnet_alpha * self.elstnet_l1_ratio * torch.norm(W, p=1) +\
0.5 * self.elstnet_alpha * (1 - self.elstnet_l1_ratio) * torch.norm(W, p=2) ** 2
else:
loss = criterion(output - b) / (SampleSize * 2) + \
self.elstnet_alpha * self.elstnet_l1_ratio * torch.norm(W, p=1) +\
0.5 * self.elstnet_alpha * (1 - self.elstnet_l1_ratio) * torch.norm(W, p=2) ** 2
elif self.method == 'ridge':
# || y - Xw|| + ||w||^2_2
if self.loss_type == 'mse' or self.loss_type == 'soft':
loss = criterion(
output,
b) + self.ridge_param * torch.norm(W, p=2)**2
else:
loss = criterion(output -
b) + self.ridge_param * torch.norm(
W, p=2)**2
elif self.method == 'ln1':
# ||w||_1
loss = torch.norm(W, p=1)
elif self.method == 'ln2':
# ||w||_2^2
loss = torch.norm(W, p=2)**2
elif self.method == 'ln12':
# ||w||_2^2 + ||w||_1
loss = torch.norm(W, p=2)**2 + torch.norm(W, p=1)
else:
raise Exception(
"Method option is wrong! Options: \'linear\', \'lasso\', \'elastic\', \'ridge\', \'ln1\', \'ln2\', \'ln12\'"
)
loss.backward()
optimizer.step()
sum_loss += loss.data.cpu().numpy()
self.loss_vec.append(loss.data.cpu().numpy())
if self.verbose:
if (epoch + 1) % self.report_step == 0:
print("Epoch: {:4d} Error: {}".format(
epoch + 1, sum_loss))
self.Beta = np.transpose(model.get_weights()[0][1].data.cpu().numpy())
self.Eps = model.get_bias()[0][1].data.cpu().numpy()
if self.gpu_enable:
A = A.cuda()
B = B.cuda()
Performance = torch.mean((B - model(A))**2) / SampleSize
Performance = Performance.data.cpu().numpy()
MSE = mean_squared_error(data_vals, np.dot(design_vals, self.Beta))
return self.Beta, self.Eps, self.loss_vec, MSE, Performance, time.time(
) - tic