def run_experiment(p, out_dir):
"""
Function to run the experiments.
p contain all the hyperparameters needed to run the experiments
We assume that all the parameters needed are present in p!!
out_dir is the out directory
#hyperparameters
"""
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#Seed
torch.manual_seed(p["seed"])
np.random.seed(p["seed"])
#Redirect output to the out dir
# sys.stdout = open(out_dir + 'output.out', 'w')
#save parameters to the out dir
with open(out_dir + "params.txt", "w") as f:
f.write(str(p))
# DEVICE
## Decidint on device on device.
DEVICE_ID = 0
DEVICE = torch.device(
'cuda:' + str(DEVICE_ID) if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
torch.cuda.set_device(DEVICE_ID)
#Tensors should have the shape
# [ntp n_ch, n_batch, n_feat]
#as n_feat can be different across channels, ntp and n_ch need to be lists. n_batch and n_feat are the tensors
X_train_pad = []
X_test_pad = []
mask_train_tensor = []
mask_test_tensor = []
#generate the data, and the mask corresponding to each channel
for ch_curves in p['curves']:
gen_model = SinDataGenerator(ch_curves,
p["ntp"],
p["noise"],
variable_tp=True)
samples = gen_model.generate_n_samples(p["nsamples"])
X_train = np.asarray([y for (_, y) in samples])
X_train_tensor = [torch.FloatTensor(t) for t in X_train]
X_train_pad_i = nn.utils.rnn.pad_sequence(X_train_tensor,
batch_first=False,
padding_value=np.nan)
mask_train = ~torch.isnan(X_train_pad_i)
mask_train_tensor.append(mask_train.to(DEVICE))
X_train_pad_i[torch.isnan(X_train_pad_i)] = 0
X_train_pad.append(X_train_pad_i.to(DEVICE))
#Do teh same for the testing set
samples = gen_model.generate_n_samples(int(p["nsamples"] * 0.8))
X_test = np.asarray([y for (_, y) in samples])
X_test_tensor = [torch.FloatTensor(t) for t in X_test]
X_test_pad_i = nn.utils.rnn.pad_sequence(X_test_tensor,
batch_first=False,
padding_value=np.nan)
mask_test = ~torch.isnan(X_test_pad_i)
mask_test_tensor.append(mask_test.to(DEVICE))
X_test_pad_i[torch.isnan(X_test_pad_i)] = 0
X_test_pad.append(X_test_pad_i.to(DEVICE))
#Stack along first dimension
#cant do that bc last dimension (features) could be different length
# X_train_tensor = torch.stack(X_train_tensor, dim=0)
#X_test_tensor = torch.stack(X_test_tensor, dim=0)
#Prepare model
# Define model and optimizer
model = rnnvae.MCRNNVAE(p["h_size"],
p["hidden"],
p["n_layers"],
p["hidden"],
p["n_layers"],
p["hidden"],
p["n_layers"],
p["z_dim"],
p["hidden"],
p["n_layers"],
p["clip"],
p["n_epochs"],
p["batch_size"],
p["n_channels"],
p["ch_type"],
p["n_feats"],
DEVICE,
print_every=100,
phi_layers=p["phi_layers"],
sigmoid_mean=p["sig_mean"],
dropout=p["dropout"],
dropout_threshold=p["drop_th"])
optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"])
model.optimizer = optimizer
model = model.to(DEVICE)
# Fit the model
model.fit(X_train_pad, X_test_pad, mask_train_tensor, mask_test_tensor)
### After training, save the model!
model.save(out_dir, 'model.pt')
# Predict the reconstructions from X_val and X_train
X_test_fwd = model.predict(X_test_pad, nt=p["ntp"])
X_train_fwd = model.predict(X_train_pad, nt=p["ntp"])
# Unpad using the masks
#plot validation and
plot_total_loss(model.loss['total'], model.val_loss['total'], "Total loss",
out_dir, "total_loss.png")
plot_total_loss(model.loss['kl'], model.val_loss['kl'], "kl_loss", out_dir,
"kl_loss.png")
plot_total_loss(model.loss['ll'], model.val_loss['ll'], "ll_loss", out_dir,
"ll_loss.png") #Negative to see downard curve
#Compute mse and reconstruction loss
#General mse and reconstruction over
test_loss = model.recon_loss(X_test_fwd,
target=X_test_pad,
mask=mask_test_tensor)
train_loss = model.recon_loss(X_train_fwd,
target=X_train_pad,
mask=mask_train_tensor)
print('MSE over the train set: ' + str(train_loss["mae"]))
print('Reconstruction loss over the train set: ' +
str(train_loss["rec_loss"]))
print('MSE over the test set: ' + str(test_loss["mae"]))
print('Reconstruction loss the train set: ' + str(test_loss["rec_loss"]))
##Latent spasce
#Reformulate things
z_train = [np.array(x).swapaxes(0, 1) for x in X_train_fwd['z']]
z_test = [np.array(x).swapaxes(0, 1) for x in X_test_fwd['z']]
# Dir for projections
proj_path = 'z_proj/'
if not os.path.exists(out_dir + proj_path):
os.makedirs(out_dir + proj_path)
#plot latent space
for ch in range(p["n_channels"]):
for dim0 in range(p["z_dim"]):
for dim1 in range(dim0, p["z_dim"]):
if dim0 == dim1: continue # very dirty
plot_z_time_2d(z_train[ch],
p["ntp"], [dim0, dim1],
out_dir + proj_path,
out_name=f'z_ch_{ch}_d{dim0}_d{dim1}')
# Dir for projections
sampling_path = 'z_proj_sampling/'
if not os.path.exists(out_dir + sampling_path):
os.makedirs(out_dir + sampling_path)
# Test the new function of latent space
qzx = [np.array(x) for x in X_train_fwd['qzx']]
# Get classificator labels, for n time points
classif = [[i] * p["nsamples"] for i in range(p["ntp"])]
classif = np.array([str(item) for elem in classif for item in elem])
print("on_classif")
print(p["ntp"])
print(len(classif))
out_dir_sample = out_dir + 'test_zspace_function/'
if not os.path.exists(out_dir_sample):
os.makedirs(out_dir_sample)
# TODO: ADAPT THIS FUNCTION TO THE
#plot_latent_space(model, qzx, p["ntp"], classificator=classif, plt_tp='all',
# all_plots=False, uncertainty=True, savefig=True, out_dir=out_dir_sample)
loss = {
"mse_train": train_loss["mae"],
"mse_test": test_loss["mae"],
"loss_total": model.loss['total'][-1],
"loss_kl": model.loss['kl'][-1],
"loss_ll": model.loss['ll'][-1]
}
return loss
def run_experiment(p, csv_path, out_dir, data_cols=[]):
"""
Function to run the experiments.
p contain all the hyperparameters needed to run the experiments
We assume that all the parameters needed are present in p!!
out_dir is the out directory
#hyperparameters
"""
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#Seed
torch.manual_seed(p["seed"])
np.random.seed(p["seed"])
#Redirect output to the out dir
# sys.stdout = open(out_dir + 'output.out', 'w')
#save parameters to the out dir
with open(out_dir + "params.txt", "w") as f:
f.write(str(p))
# DEVICE
## Decidint on device on device.
DEVICE_ID = 0
DEVICE = torch.device(
'cuda:' + str(DEVICE_ID) if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
torch.cuda.set_device(DEVICE_ID)
# LOAD DATA
#Start by not using validation data
# this is a list of values
X_train, X_test, Y_train, Y_test, mri_col = load_multimodal_data(
csv_path,
data_cols,
p["ch_type"],
train_set=0.9,
normalize=True,
return_covariates=True)
p["n_feats"] = [x[0].shape[1] for x in X_train]
X_train_list = []
mask_train_list = []
X_test_list = []
mask_test_list = []
print('Length of train/test')
print(len(X_train[0]))
print(len(X_test[0]))
#For each channel, pad, create the mask, and append
for x_ch in X_train:
X_train_tensor = [torch.FloatTensor(t) for t in x_ch]
X_train_pad = nn.utils.rnn.pad_sequence(X_train_tensor,
batch_first=False,
padding_value=np.nan)
mask_train = ~torch.isnan(X_train_pad)
mask_train_list.append(mask_train.to(DEVICE))
X_train_pad[torch.isnan(X_train_pad)] = 0
X_train_list.append(X_train_pad.to(DEVICE))
for x_ch in X_test:
X_test_tensor = [torch.FloatTensor(t) for t in x_ch]
X_test_pad = nn.utils.rnn.pad_sequence(X_test_tensor,
batch_first=False,
padding_value=np.nan)
mask_test = ~torch.isnan(X_test_pad)
mask_test_list.append(mask_test.to(DEVICE))
X_test_pad[torch.isnan(X_test_pad)] = 0
X_test_list.append(X_test_pad.to(DEVICE))
# ntp = max(X_train_list[0].shape[0], X_test_list[0].shape[0])
ntp = max(max([x.shape[0] for x in X_train_list]),
max([x.shape[0] for x in X_train_list]))
model = rnnvae_h.MCRNNVAE(p["h_size"],
p["hidden"],
p["n_layers"],
p["hidden"],
p["n_layers"],
p["hidden"],
p["n_layers"],
p["z_dim"],
p["hidden"],
p["n_layers"],
p["clip"],
p["n_epochs"],
p["batch_size"],
p["n_channels"],
p["ch_type"],
p["n_feats"],
DEVICE,
print_every=100,
phi_layers=p["phi_layers"],
sigmoid_mean=p["sig_mean"],
dropout=p["dropout"],
dropout_threshold=p["drop_th"])
model.ch_name = p["ch_names"]
optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"])
model.optimizer = optimizer
model = model.to(DEVICE)
# Fit the model
model.fit(X_train_list, X_test_list, mask_train_list, mask_test_list)
#fit the model after changing the lr
if p["dropout"]:
print("Print the dropout")
print(model.dropout_comp)
### After training, save the model!
model.save(out_dir, 'model.pt')
# Predict the reconstructions from X_val and X_train
X_train_fwd = model.predict(X_train_list, mask_train_list, nt=ntp)
X_test_fwd = model.predict(X_test_list, mask_test_list, nt=ntp)
# Unpad using the masks
#plot validation and
plot_total_loss(model.loss['total'], model.val_loss['total'], "Total loss",
out_dir, "total_loss.png")
plot_total_loss(model.loss['kl'], model.val_loss['kl'], "kl_loss", out_dir,
"kl_loss.png")
plot_total_loss(model.loss['ll'], model.val_loss['ll'], "ll_loss", out_dir,
"ll_loss.png") #Negative to see downard curve
#Compute mse and reconstruction loss
#General mse and reconstruction over
# test_loss = model.recon_loss(X_test_fwd, target=X_test_pad, mask=mask_test_tensor)
train_loss = model.recon_loss(X_train_fwd,
target=X_train_list,
mask=mask_train_list)
test_loss = model.recon_loss(X_test_fwd,
target=X_test_list,
mask=mask_test_list)
print('MSE over the train set: ' + str(train_loss["mae"]))
print('Reconstruction loss over the train set: ' +
str(train_loss["rec_loss"]))
print('MSE over the test set: ' + str(test_loss["mae"]))
print('Reconstruction loss the train set: ' + str(test_loss["rec_loss"]))
pred_results = {}
for ch_name in p["ch_names"][:3]:
pred_results[f"pred_{ch_name}_mae"] = []
rec_results = {}
for ch_name in p["ch_names"]:
rec_results[f"recon_{ch_name}_mae"] = []
results = {**pred_results, **rec_results}
######################
## Prediction of last time point
######################
# FUTURE TWO TP
X_test_list_minus = []
X_test_tensors = []
mask_test_list_minus = []
for x_ch in X_test:
X_test_tensor = [torch.FloatTensor(t[:-1, :]) for t in x_ch]
X_test_tensor_full = [torch.FloatTensor(t) for t in x_ch]
X_test_tensors.append(X_test_tensor_full)
X_test_pad = nn.utils.rnn.pad_sequence(X_test_tensor,
batch_first=False,
padding_value=np.nan)
mask_test = ~torch.isnan(X_test_pad)
mask_test_list_minus.append(mask_test.to(DEVICE))
X_test_pad[torch.isnan(X_test_pad)] = 0
X_test_list_minus.append(X_test_pad.to(DEVICE))
# Run prediction
#this is terribly programmed holy shit
X_test_fwd_minus = model.predict(X_test_list_minus,
mask_test_list_minus,
nt=ntp)
X_test_xnext = X_test_fwd_minus["xnext"]
# Test data without last timepoint
# X_test_tensors do have the last timepoint
i = 0
# import pdb; pdb.set_trace()
for (X_ch, ch) in zip(X_test[:3], p["ch_names"][:3]):
#Select a single channel
print(f'testing for {ch}')
y_true = [x[-1] for x in X_ch if len(x) > 1]
last_tp = [len(x) - 1 for x in X_ch
] # last tp is max size of original data minus one
y_pred = []
# for each subject, select last tp
j = 0
for tp in last_tp:
if tp < 1:
j += 1
continue # ignore tps with only baseline
y_pred.append(X_test_xnext[i][tp, j, :])
j += 1
#Process it to predict it
mae_tp_ch = mean_absolute_error(y_true, y_pred)
#save the result
results[f'pred_{ch}_mae'] = mae_tp_ch
i += 1
############################
## Test reconstruction for each channel, using the other one
############################
# For each channel
if p["n_channels"] > 1:
for i in range(len(X_test)):
curr_name = p["ch_names"][i]
av_ch = list(range(len(X_test)))
av_ch.remove(i)
# try to reconstruct it from the other ones
ch_recon = model.predict(X_test_list,
mask_test_list,
nt=ntp,
av_ch=av_ch,
task='recon')
#for all existing timepoints
y_true = X_test[i]
# swap dims to iterate over subjects
y_pred = np.transpose(ch_recon["xnext"][i], (1, 0, 2))
y_pred = [
x_pred[:len(x_true)]
for (x_pred, x_true) in zip(y_pred, y_true)
]
#prepare it timepoint wise
y_pred = [tp for subj in y_pred for tp in subj]
y_true = [tp for subj in y_true for tp in subj]
mae_rec_ch = mean_absolute_error(y_true, y_pred)
# Get MAE result for that specific channel over all timepoints
results[f"recon_{curr_name}_mae"] = mae_rec_ch
loss = {
"mae_train": train_loss["mae"],
"rec_train": train_loss["rec_loss"],
"mae_test": test_loss["mae"],
"loss_total": model.loss['total'][-1],
"loss_kl": model.loss['kl'][-1],
"loss_ll": model.loss['ll'][-1],
}
if p["dropout"]:
loss["dropout_comps"] = model.dropout_comp
loss = {**loss, **results}
print(loss)
# Dir for projections
proj_path = 'z_proj/'
if not os.path.exists(out_dir + proj_path):
os.makedirs(out_dir + proj_path)
# Test the new function of latent space
#NEED TO ADAPT THIS FUNCTION
qzx_train = [np.array(x) for x in X_train_fwd['qzx']]
qzx_test = [np.array(x) for x in X_test_fwd['qzx']]
#Convert to standard
#Add padding so that the mask also works here
DX_train = [[x for x in elem] for elem in Y_train["DX"]]
DX_test = [[x for x in elem] for elem in Y_test["DX"]]
#Define colors
pallete_dict = {"CN": "#2a9e1e", "MCI": "#bfbc1a", "AD": "#af1f1f"}
# Get classificator labels, for n time points
out_dir_sample = out_dir + 'zcomp_ch_dx/'
if not os.path.exists(out_dir_sample):
os.makedirs(out_dir_sample)
plot_latent_space(model,
qzx_test,
ntp,
classificator=DX_test,
pallete_dict=pallete_dict,
plt_tp='all',
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample + '_test',
mask=mask_test_list)
plot_latent_space(model,
qzx_train,
ntp,
classificator=DX_train,
pallete_dict=pallete_dict,
plt_tp='all',
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample + '_train',
mask=mask_train_list)
out_dir_sample_t0 = out_dir + 'zcomp_ch_dx_t0/'
if not os.path.exists(out_dir_sample_t0):
os.makedirs(out_dir_sample_t0)
plot_latent_space(model,
qzx_train,
ntp,
classificator=DX_train,
pallete_dict=pallete_dict,
plt_tp=[0],
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample_t0 + '_train',
mask=mask_train_list)
plot_latent_space(model,
qzx_test,
ntp,
classificator=DX_test,
pallete_dict=pallete_dict,
plt_tp=[0],
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample_t0 + '_test',
mask=mask_test_list)
# Now plot color by timepoint
out_dir_sample = out_dir + 'zcomp_ch_tp/'
if not os.path.exists(out_dir_sample):
os.makedirs(out_dir_sample)
classif_train = [[i for (i, x) in enumerate(elem)]
for elem in Y_train["DX"]]
classif_test = [[i for (i, x) in enumerate(elem)] for elem in Y_test["DX"]]
pallete = sns.color_palette("viridis", ntp)
pallete_dict = {i: value for (i, value) in enumerate(pallete)}
plot_latent_space(model,
qzx_train,
ntp,
classificator=classif_train,
pallete_dict=pallete_dict,
plt_tp='all',
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample + '_train',
mask=mask_train_list)
plot_latent_space(model,
qzx_test,
ntp,
classificator=classif_test,
pallete_dict=pallete_dict,
plt_tp='all',
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample + '_test',
mask=mask_test_list)
return loss
def run_experiment(p, csv_path, out_dir, data_cols=[]):
"""
Function to run the experiments.
p contain all the hyperparameters needed to run the experiments
We assume that all the parameters needed are present in p!!
out_dir is the out directory
#hyperparameters
"""
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#Seed
torch.manual_seed(p["seed"])
np.random.seed(p["seed"])
#Redirect output to the out dir
sys.stdout = open(out_dir + 'output.out', 'w')
#save parameters to the out dir
with open(out_dir + "params.txt", "w") as f:
f.write(str(p))
# DEVICE
## Decidint on device on device.
DEVICE_ID = 0
DEVICE = torch.device(
'cuda:' + str(DEVICE_ID) if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
torch.cuda.set_device(DEVICE_ID)
# Begin on the CV data
gen = load_multimodal_data_cv(csv_path,
data_cols,
p["ch_type"],
nsplit=10,
normalize=True)
# Prepare the data structures for the data
# Load the different folds
loss = {
"mae_train": [],
"rec_train": [],
"mae_test": [],
"loss_total": [],
"loss_total_val": [],
"loss_kl": [],
"loss_ll": [],
}
pred_results = {}
for ch_name in p["ch_names"][:3]:
pred_results[f"pred_{ch_name}_mae"] = []
rec_results = {}
for ch_name in p["ch_names"]:
rec_results[f"recon_{ch_name}_mae"] = []
loss = {**loss, **pred_results, **rec_results}
# iterator marking the fold
fold_n = 0
for X_train, X_test, Y_train, Y_test, mri_col in gen:
# LOAD DATA
#Start by not using validation data
# this is a list of values
#Create output dir for the fold
out_dir_cv = out_dir + f'_fold_{fold_n}/'
if not os.path.exists(out_dir_cv):
os.makedirs(out_dir_cv)
#Redirect output to specific folder
sys.stdout = open(out_dir_cv + 'output.out', 'w')
p["n_feats"] = [x[0].shape[1] for x in X_train]
X_train_list = []
mask_train_list = []
X_test_list = []
mask_test_list = []
print('Length of train/test')
print(len(X_train[0]))
print(len(X_test[0]))
# need to deal with ntp here
ntp = max(np.max([[len(xi) for xi in x] for x in X_train]),
np.max([[len(xi) for xi in x] for x in X_train]))
if p["long_to_bl"]:
# HERE, change bl to long and repeat the values at t0 for ntp
for i in range(len(p["ch_type"])):
if p["ch_type"][i] == 'bl':
for j in range(len(X_train[i])):
X_train[i][j] = np.array([X_train[i][j][0]] * ntp)
for j in range(len(X_test[i])):
X_test[i][j] = np.array([X_test[i][j][0]] * ntp)
# p["ch_type"][i] = 'long'
#For each channel, pad, create the mask, and append
for x_ch in X_train:
X_train_tensor = [torch.FloatTensor(t) for t in x_ch]
X_train_pad = nn.utils.rnn.pad_sequence(X_train_tensor,
batch_first=False,
padding_value=np.nan)
mask_train = ~torch.isnan(X_train_pad)
mask_train_list.append(mask_train.to(DEVICE))
X_train_pad[torch.isnan(X_train_pad)] = 0
X_train_list.append(X_train_pad.to(DEVICE))
for x_ch in X_test:
X_test_tensor = [torch.FloatTensor(t) for t in x_ch]
X_test_pad = nn.utils.rnn.pad_sequence(X_test_tensor,
batch_first=False,
padding_value=np.nan)
mask_test = ~torch.isnan(X_test_pad)
mask_test_list.append(mask_test.to(DEVICE))
X_test_pad[torch.isnan(X_test_pad)] = 0
X_test_list.append(X_test_pad.to(DEVICE))
#ntp = max(max([x.shape[0] for x in X_train_list]), max([x.shape[0] for x in X_train_list]))
model = rnnvae_h.MCRNNVAE(p["h_size"],
p["x_hidden"],
p["x_n_layers"],
p["z_hidden"],
p["z_n_layers"],
p["enc_hidden"],
p["enc_n_layers"],
p["z_dim"],
p["dec_hidden"],
p["dec_n_layers"],
p["clip"],
p["n_epochs"],
p["batch_size"],
p["n_channels"],
p["ch_type"],
p["n_feats"],
p["c_z"],
DEVICE,
print_every=100,
phi_layers=p["phi_layers"],
sigmoid_mean=p["sig_mean"],
dropout=p["dropout"],
dropout_threshold=p["drop_th"])
model.ch_name = p["ch_names"]
optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"])
model.optimizer = optimizer
model = model.to(DEVICE)
# Fit the model
model.fit(X_train_list, X_test_list, mask_train_list, mask_test_list)
#fit the model after changing the lr
#optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"]*.1)
#model.optimizer = optimizer
#print('Refining optimization...')
#model.fit(X_train_list, X_test_list, mask_train_list, mask_test_list)
if p["dropout"]:
print("Print the dropout")
print(model.dropout_comp)
### After training, save the model!
model.save(out_dir_cv, 'model.pt')
# Predict the reconstructions from X_val and X_train
X_train_fwd = model.predict(X_train_list, mask_train_list, nt=ntp)
X_test_fwd = model.predict(X_test_list, mask_test_list, nt=ntp)
# Unpad using the masks
#plot validation and
plot_total_loss(model.loss['total'], model.val_loss['total'],
"Total loss", out_dir_cv, "total_loss.png")
plot_total_loss(model.loss['kl'], model.val_loss['kl'], "kl_loss",
out_dir_cv, "kl_loss.png")
plot_total_loss(model.loss['ll'], model.val_loss['ll'], "ll_loss",
out_dir_cv,
"ll_loss.png") #Negative to see downard curve
#Compute mse and reconstruction loss
#General mse and reconstruction over
# test_loss = model.recon_loss(X_test_fwd, target=X_test_pad, mask=mask_test_tensor)
train_loss = model.recon_loss(X_train_fwd,
target=X_train_list,
mask=mask_train_list)
test_loss = model.recon_loss(X_test_fwd,
target=X_test_list,
mask=mask_test_list)
print('MSE over the train set: ' + str(train_loss["mae"]))
print('Reconstruction loss over the train set: ' +
str(train_loss["rec_loss"]))
print('MSE over the test set: ' + str(test_loss["mae"]))
print('Reconstruction loss the train set: ' +
str(test_loss["rec_loss"]))
######################
## Prediction of last time point
######################
i = 0
# Test data without last timepoint
# X_test_tensors do have the last timepoint
pred_ch = list(range(3))
print(pred_ch)
t_pred = 1
res = eval_prediction(model, X_test, t_pred, pred_ch, DEVICE)
for (i, ch) in enumerate(
[x for (i, x) in enumerate(p["ch_names"]) if i in pred_ch]):
loss[f'pred_{ch}_mae'].append(res[i])
############################
## Test reconstruction for each channel, using the other one
############################
# For each channel
if p["n_channels"] > 1:
for i in range(len(X_test)):
curr_name = p["ch_names"][i]
av_ch = list(range(len(X_test)))
av_ch.remove(i)
mae_rec = eval_reconstruction(model, X_test, X_test_list,
mask_test_list, av_ch, i)
# Get MAE result for that specific channel over all timepoints
loss[f"recon_{curr_name}_mae"].append(mae_rec)
# Save results in the loss object
loss["mae_train"].append(train_loss["mae"])
loss["rec_train"].append(train_loss["rec_loss"])
loss["mae_test"].append(train_loss["mae"])
loss["loss_total"].append(model.loss['total'][-1])
loss["loss_total_val"].append(model.val_loss['total'][-1])
loss["loss_kl"].append(model.loss['kl'][-1])
loss["loss_ll"].append(model.loss['ll'][-1])
fold_n += 1
# break at 5 iterations, need to do it faster
if fold_n == 2:
break
# Compute the mean for every param in the loss dict
for k in loss.keys():
loss[k] = np.mean(loss[k])
print(loss)
return loss
def run_experiment(p, csv_path, out_dir, data_cols=[]):
"""
Function to run the experiments.
p contain all the hyperparameters needed to run the experiments
We assume that all the parameters needed are present in p!!
out_dir is the out directory
#hyperparameters
"""
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#Seed
torch.manual_seed(p["seed"])
np.random.seed(p["seed"])
#Redirect output to the out dir
# sys.stdout = open(out_dir + 'output.out', 'w')
#save parameters to the out dir
with open(out_dir + "params.txt", "w") as f:
f.write(str(p))
# DEVICE
## Decidint on device on device.
DEVICE_ID = 0
DEVICE = torch.device(
'cuda:' + str(DEVICE_ID) if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
torch.cuda.set_device(DEVICE_ID)
# LOAD DATA
#Start by not using validation data
# this is a list of values
X_train, X_test, Y_train, Y_test, mri_col = load_multimodal_data(
csv_path,
data_cols,
train_set=0.8,
normalize=True,
return_covariates=True)
p["n_feats"] = [x[0].shape[1] for x in X_train]
X_train_list = []
mask_train_list = []
X_test_list = []
mask_test_list = []
print('Length of train/test')
print(len(X_train[0]))
print(len(X_test[0]))
#For each channel, pad, create the mask, and append
for x_ch in X_train:
X_train_tensor = [torch.FloatTensor(t) for t in x_ch]
X_train_pad = nn.utils.rnn.pad_sequence(X_train_tensor,
batch_first=False,
padding_value=np.nan)
mask_train = ~torch.isnan(X_train_pad)
mask_train_list.append(mask_train.to(DEVICE))
X_train_pad[torch.isnan(X_train_pad)] = 0
X_train_list.append(X_train_pad.to(DEVICE))
for x_ch in X_test:
X_test_tensor = [torch.FloatTensor(t) for t in x_ch]
X_test_pad = nn.utils.rnn.pad_sequence(X_test_tensor,
batch_first=False,
padding_value=np.nan)
mask_test = ~torch.isnan(X_test_pad)
mask_test_list.append(mask_test.to(DEVICE))
X_test_pad[torch.isnan(X_test_pad)] = 0
X_test_list.append(X_test_pad.to(DEVICE))
# ntp = max(X_train_list[0].shape[0], X_test_list[0].shape[0])
ntp = max(max([x.shape[0] for x in X_train_list]),
max([x.shape[0] for x in X_train_list]))
model = rnnvae_drop.MCRNNVAE(p["h_size"],
p["hidden"],
p["n_layers"],
p["hidden"],
p["n_layers"],
p["hidden"],
p["n_layers"],
p["z_dim"],
p["hidden"],
p["n_layers"],
p["clip"],
p["n_epochs"],
p["batch_size"],
p["n_channels"],
p["n_feats"],
DEVICE,
0.3,
print_every=100)
model.ch_name = p["ch_names"]
optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"])
model.optimizer = optimizer
model = model.to(DEVICE)
# Fit the model
model.fit(X_train_list, X_test_list, mask_train_list, mask_test_list)
print("Print the dropout")
print(model.dropout)
### After training, save the model!
model.save(out_dir, 'model.pt')
# Predict the reconstructions from X_val and X_train
X_train_fwd = model.predict(X_train_list, nt=ntp)
X_test_fwd = model.predict(X_test_list, nt=ntp)
# Unpad using the masks
#plot validation and
plot_total_loss(model.loss['total'], model.val_loss['total'], "Total loss",
out_dir, "total_loss.png")
plot_total_loss(model.loss['kl'], model.val_loss['kl'], "kl_loss", out_dir,
"kl_loss.png")
plot_total_loss(model.loss['ll'], model.val_loss['ll'], "ll_loss", out_dir,
"ll_loss.png") #Negative to see downard curve
#Compute mse and reconstruction loss
#General mse and reconstruction over
# test_loss = model.recon_loss(X_test_fwd, target=X_test_pad, mask=mask_test_tensor)
train_loss = model.recon_loss(X_train_fwd,
target=X_train_list,
mask=mask_train_list)
test_loss = model.recon_loss(X_test_fwd,
target=X_test_list,
mask=mask_test_list)
print('MSE over the train set: ' + str(train_loss["mae"]))
print('Reconstruction loss over the train set: ' +
str(train_loss["rec_loss"]))
print('MSE over the test set: ' + str(test_loss["mae"]))
print('Reconstruction loss the train set: ' + str(test_loss["rec_loss"]))
######################
## Prediction of last time point
######################
# Test data without last timepoint
# X_test_tensors do have the last timepoint
X_test_list_minus = []
X_test_tensors = []
mask_test_list_minus = []
for x_ch in X_test:
X_test_tensor = [torch.FloatTensor(t[:-1, :]) for t in x_ch]
X_test_tensor_full = [torch.FloatTensor(t) for t in x_ch]
X_test_tensors.append(X_test_tensor_full)
X_test_pad = nn.utils.rnn.pad_sequence(X_test_tensor,
batch_first=False,
padding_value=np.nan)
mask_test = ~torch.isnan(X_test_pad)
mask_test_list_minus.append(mask_test.to(DEVICE))
X_test_pad[torch.isnan(X_test_pad)] = 0
X_test_list_minus.append(X_test_pad.to(DEVICE))
# Run prediction
#this is terribly programmed holy shit
X_test_fwd_minus = model.predict(X_test_list_minus, nt=ntp)
X_test_xnext = X_test_fwd_minus["xnext"]
last_tp_mse = 0
#for each channel
for i in range(len(X_test_tensors)):
#For each subject, select the tp of the mask
last_tp_mse_ch = 0
n_mae = 0
for j in range(len(X_test_tensors[i])):
tp = len(X_test_tensors[i][j]) - 1
last_tp_mse_ch += mean_squared_error(X_test_tensors[i][j][tp, :],
X_test_xnext[i][tp, j, :])
n_mae += 1
#compute the mean
last_tp_mse += last_tp_mse_ch / n_mae
#Compute MAE over last timepoint
############################
## Test reconstruction for each channel, using the other one
############################
# For each channel
rec_results = {}
for i in range(len(X_test_list)):
curr_name = p["ch_names"][i]
av_ch = list(range(len(X_test_list)))
av_ch.remove(i)
# try to reconstruct it from the other ones
ch_recon = model.predict(X_test_list, nt=ntp, av_ch=av_ch)
ch_recon["xnext"]
#for all existing timepoints
mae_loss = 0
for t in range(len(mask_test_list[i])):
mask_channel = mask_test_list[i][t, :, 0]
mae_loss += rnnvae_drop.mae(target=X_test_list[i][t].cpu(),
predicted=ch_recon["xnext"][i][t],
mask=mask_channel)
# Get MAE result for that specific channel over all timepoints
#for this, i also need the mask
rec_results[f"recon_{curr_name}_mae"] = mae_loss.item()
# Dir for projections
proj_path = 'z_proj/'
if not os.path.exists(out_dir + proj_path):
os.makedirs(out_dir + proj_path)
# Test the new function of latent space
#NEED TO ADAPT THIS FUNCTION
qzx_train = [np.array(x) for x in X_train_fwd['qzx']]
qzx_test = [np.array(x) for x in X_test_fwd['qzx']]
#Convert to standard
#Add padding so that the mask also works here
DX_train = [[x for x in elem] for elem in Y_train["DX"]]
DX_test = [[x for x in elem] for elem in Y_test["DX"]]
#Define colors
pallete_dict = {"CN": "#2a9e1e", "MCI": "#bfbc1a", "AD": "#af1f1f"}
# Get classificator labels, for n time points
out_dir_sample = out_dir + 'zcomp_ch_dx/'
if not os.path.exists(out_dir_sample):
os.makedirs(out_dir_sample)
plot_latent_space(model,
qzx_test,
ntp,
classificator=DX_test,
pallete_dict=pallete_dict,
plt_tp='all',
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample + '_test',
mask=mask_test_list)
plot_latent_space(model,
qzx_train,
ntp,
classificator=DX_train,
pallete_dict=pallete_dict,
plt_tp='all',
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample + '_train',
mask=mask_train_list)
out_dir_sample_t0 = out_dir + 'zcomp_ch_dx_t0/'
if not os.path.exists(out_dir_sample_t0):
os.makedirs(out_dir_sample_t0)
plot_latent_space(model,
qzx_train,
ntp,
classificator=DX_train,
pallete_dict=pallete_dict,
plt_tp=[0],
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample_t0 + '_train',
mask=mask_train_list)
plot_latent_space(model,
qzx_test,
ntp,
classificator=DX_test,
pallete_dict=pallete_dict,
plt_tp=[0],
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample_t0 + '_test',
mask=mask_test_list)
# Now plot color by timepoint
out_dir_sample = out_dir + 'zcomp_ch_tp/'
if not os.path.exists(out_dir_sample):
os.makedirs(out_dir_sample)
classif_train = [[i for (i, x) in enumerate(elem)]
for elem in Y_train["DX"]]
classif_test = [[i for (i, x) in enumerate(elem)] for elem in Y_test["DX"]]
pallete = sns.color_palette("viridis", ntp)
pallete_dict = {i: value for (i, value) in enumerate(pallete)}
plot_latent_space(model,
qzx_train,
ntp,
classificator=classif_train,
pallete_dict=pallete_dict,
plt_tp='all',
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample + '_train',
mask=mask_train_list)
plot_latent_space(model,
qzx_test,
ntp,
classificator=classif_test,
pallete_dict=pallete_dict,
plt_tp='all',
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample + '_test',
mask=mask_test_list)
loss = {
"mae_train": train_loss["mae"],
"rec_train": train_loss["rec_loss"],
"mae_test": test_loss["mae"],
"mae_last_tp": last_tp_mse,
"loss_total": model.loss['total'][-1],
"loss_kl": model.loss['kl'][-1],
"loss_ll": model.loss['ll'][-1]
}
loss = {**loss, **rec_results}
return loss
model = rnnvae_h.MCRNNVAE(p["h_size"], p["hidden"], p["n_layers"],
p["hidden"], p["n_layers"], p["hidden"],
p["n_layers"], p["z_dim"], p["hidden"], p["n_layers"],
p["clip"], p["n_epochs"], p["batch_size"],
p["n_channels"], p["ch_type"], p["n_feats"], DEVICE, print_every=100,
phi_layers=p["phi_layers"], sigmoid_mean=p["sig_mean"],
dropout=p["dropout"], dropout_threshold=p["drop_th"])
optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"])
model.optimizer = optimizer
model = model.to(DEVICE)
# Fit the model
model.fit(X_train_list, X_train_list, mask_train_list, mask_train_list)
### After training, save the model!
model.save(out_dir, 'model.pt')
if p["dropout"]:
print("Print the dropout")
print(model.dropout_comp)
# Predict the reconstructions from X_val and X_train
X_train_fwd = model.predict(X_train_list, mask_train_list, nt=ntp)
# Unpad using the masks
#plot validation and tal
plot_total_loss(model.loss['total'], model.val_loss['total'], "Total loss", out_dir, "total_loss.png")
plot_total_loss(model.loss['kl'], model.val_loss['kl'], "kl_loss", out_dir, "kl_loss.png")
plot_total_loss(model.loss['ll'], model.val_loss['ll'], "ll_loss", out_dir, "ll_loss.png") #Negative to see downard curve
def run_experiment(p, csv_path, out_dir):
"""
Function to run the experiments.
p contain all the hyperparameters needed to run the experiments
We assume that all the parameters needed are present in p!!
out_dir is the out directory
#hyperparameters
"""
# out_dir
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#Seed
torch.manual_seed(p["seed"])
np.random.seed(p["seed"])
#Redirect output to the out dir
sys.stdout = open(out_dir + 'output.out', 'w')
#save parameters to the out dir
with open(out_dir + "params.txt", "w") as f:
f.write(str(p))
# DEVICE
## Decidint on device on device.
DEVICE_ID = 0
DEVICE = torch.device(
'cuda:' + str(DEVICE_ID) if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
torch.cuda.set_device(DEVICE_ID)
# LOAD DATA
X_train, X_test, Y_train, Y_test = open_MRI_data(csv_path,
train_set=0.9,
normalize=True,
return_covariates=True)
#Combine test and train Y for later
Y = {}
for k in Y_train.keys():
Y[k] = Y_train[k] + Y_test[k]
# List of (nt, nfeatures) numpy objects
p["x_size"] = X_train[0].shape[1]
print(p["x_size"])
# Apply padding to both X_train and X_val
#Permute so that dimension is (tp, nbatch, feat)
X_train_tensor = torch.FloatTensor(X_train).permute((1, 0, 2))
X_test_tensor = torch.FloatTensor(X_test).permute((1, 0, 2))
# Define model and optimizer
model = rnnvae.ModelRNNVAE(p["x_size"], p["h_size"], p["hidden"],
p["n_layers"], p["hidden"], p["n_layers"],
p["hidden"], p["n_layers"], p["z_dim"],
p["hidden"], p["n_layers"], p["clip"],
p["n_epochs"], p["batch_size"], DEVICE)
optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"])
model.optimizer = optimizer
model = model.to(DEVICE)
# Fit the model
model.fit(X_train_tensor.to(DEVICE), X_test_tensor.to(DEVICE))
### After training, save the model!
model.save(out_dir, 'model.pt')
# Predict the reconstructions from X_val and X_train
X_test_fwd = model.predict(X_test_tensor.to(DEVICE))
X_train_fwd = model.predict(X_train_tensor.to(DEVICE))
#Reformulate things
X_train_fwd['xnext'] = np.array(X_train_fwd['xnext']).swapaxes(0, 1)
X_train_fwd['z'] = np.array(X_train_fwd['z']).swapaxes(0, 1)
X_test_fwd['xnext'] = np.array(X_test_fwd['xnext']).swapaxes(0, 1)
X_test_fwd['z'] = np.array(X_test_fwd['z']).swapaxes(0, 1)
X_test_hat = X_test_fwd["xnext"]
X_train_hat = X_train_fwd["xnext"]
# Unpad using the masks
#after masking, need to rehsape to (nt, nfeat)
#Compute mean absolute error over all sequences
mse_train = np.mean([
mean_absolute_error(xval, xhat)
for (xval, xhat) in zip(X_train, X_train_hat)
])
print('MSE over the train set: ' + str(mse_train))
#Compute mean absolute error over all sequences
mse_test = np.mean([
mean_absolute_error(xval, xhat)
for (xval, xhat) in zip(X_test, X_test_hat)
])
print('MSE over the test set: ' + str(mse_test))
#plot validation and
plot_total_loss(model.loss['total'], model.val_loss['total'], "Total loss",
out_dir, "total_loss.png")
plot_total_loss(model.loss['kl'], model.val_loss['kl'], "kl_loss", out_dir,
"kl_loss.png")
plot_total_loss(model.loss['ll'], model.val_loss['ll'], "ll_loss", out_dir,
"ll_loss.png") #Negative to see downard curve
# Visualization of trajectories
subj = 6
feature = 12
# For train
plot_trajectory(
X_train, X_train_hat, subj, 'all', out_dir,
f'traj_train_s_{subj}_f_all') # testing for a given subject
plot_trajectory(
X_train, X_train_hat, subj, feature, out_dir,
f'traj_train_s_{subj}_f_{feature}') # testing for a given feature
# For test
plot_trajectory(X_test, X_test_hat, subj, 'all', out_dir,
f'traj_test_s_{subj}_f_all') # testing for a given subject
plot_trajectory(
X_test, X_test_hat, subj, feature, out_dir,
f'traj_test_s_{subj}_f_{feature}') # testing for a given feature
z_train = X_train_fwd['z']
z_test = X_test_fwd['z']
z_train = [Z for (i, Z) in enumerate(z_train)]
z_test = [Z for (i, Z) in enumerate(z_test)]
z = z_train + z_test
# Dir for projections
proj_path = 'z_proj/'
if not os.path.exists(out_dir + proj_path):
os.makedirs(out_dir + proj_path)
#plot latent space
for dim0 in range(p["z_dim"]):
for dim1 in range(dim0, p["z_dim"]):
if dim0 == dim1: continue # very dirty
plot_z_time_2d(z,
p["ntp"], [dim0, dim1],
out_dir + 'z_proj/',
out_name=f'z_d{dim0}_d{dim1}')
# Dir for projections
sampling_path = 'z_proj_dx/'
if not os.path.exists(out_dir + sampling_path):
os.makedirs(out_dir + sampling_path)
#plot latent space
for dim0 in range(p["z_dim"]):
for dim1 in range(dim0, p["z_dim"]):
if dim0 == dim1: continue # very dirty
plot_z_time_2d(z,
p["ntp"], [dim0, dim1],
out_dir + sampling_path,
c='DX',
Y=Y,
out_name=f'z_d{dim0}_d{dim1}')
# Dir for projections
sampling_path = 'z_proj_age/'
if not os.path.exists(out_dir + sampling_path):
os.makedirs(out_dir + sampling_path)
#plot latent space
for dim0 in range(p["z_dim"]):
for dim1 in range(dim0, p["z_dim"]):
if dim0 == dim1: continue # very dirty
plot_z_time_2d(z,
p["ntp"], [dim0, dim1],
out_dir + sampling_path,
c='AGE',
Y=Y,
out_name=f'z_d{dim0}_d{dim1}')
#Sampling
# TODO: THIS NEEDS TO BE UPDATED
nt = p['ntp']
nsamples = 1000
X_sample = model.sample_latent(nsamples, nt)
#Get the samples
X_sample['xnext'] = np.array(X_sample['xnext']).swapaxes(0, 1)
X_sample['z'] = np.array(X_sample['z']).swapaxes(0, 1)
# Dir for projections
sampling_path = 'z_proj_sampling/'
if not os.path.exists(out_dir + sampling_path):
os.makedirs(out_dir + sampling_path)
#plot latent space
for dim0 in range(p["z_dim"]):
for dim1 in range(dim0, p["z_dim"]):
if dim0 == dim1: continue # very dirty
plot_z_time_2d(X_sample['z'],
p["ntp"], [dim0, dim1],
out_dir + 'z_proj_sampling/',
out_name=f'z_d{dim0}_d{dim1}')
loss = {
"mse_train": mse_train,
"mse_test": mse_test,
"loss_total": model.loss['total'][-1],
"loss_kl": model.loss['kl'][-1],
"loss_ll": model.loss['ll'][-1]
}
return loss
def run_experiment(p, out_dir):
"""
Function to run the experiments.
p contain all the hyperparameters needed to run the experiments
We assume that all the parameters needed are present in p!!
out_dir is the out directory
#hyperparameters
"""
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#Seed
torch.manual_seed(p["seed"])
np.random.seed(p["seed"])
#Redirect output to the out dir
sys.stdout = open(out_dir + 'output.out', 'w')
#save parameters to the out dir
with open(out_dir + "params.txt", "w") as f:
f.write(str(p))
# DEVICE
## Decidint on device on device.
DEVICE_ID = 0
DEVICE = torch.device(
'cuda:' + str(DEVICE_ID) if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
torch.cuda.set_device(DEVICE_ID)
#generate the data
#Create class object
gen_model = SinDataGenerator(p["curves"], p["ntp"], p["noise"])
samples = gen_model.generate_n_samples(p["nsamples"])
X_train = np.asarray([y for (_, y) in samples])
X_train_tensor = torch.FloatTensor(X_train).permute((1, 0, 2))
samples = gen_model.generate_n_samples(int(p["nsamples"] * 0.8))
X_test = np.asarray([y for (_, y) in samples])
X_test_tensor = torch.FloatTensor(X_test).permute((1, 0, 2))
#Prepare model
# Define model and optimizer
model = rnnvae.ModelRNNVAE(p["x_size"], p["h_size"], p["hidden"],
p["n_layers"], p["hidden"], p["n_layers"],
p["hidden"], p["n_layers"], p["z_dim"],
p["hidden"], p["n_layers"], p["clip"],
p["n_epochs"], p["batch_size"], DEVICE)
optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"])
model.optimizer = optimizer
model = model.to(DEVICE)
# Fit the model
model.fit(X_train_tensor.to(DEVICE), X_test_tensor.to(DEVICE))
### After training, save the model!
model.save(out_dir, 'model.pt')
# Predict the reconstructions from X_val and X_train
X_test_fwd = model.predict(X_test_tensor.to(DEVICE))
X_train_fwd = model.predict(X_train_tensor.to(DEVICE))
#Reformulate things
X_train_fwd['xnext'] = np.array(X_train_fwd['xnext']).swapaxes(0, 1)
X_train_fwd['z'] = np.array(X_train_fwd['z']).swapaxes(0, 1)
X_test_fwd['xnext'] = np.array(X_test_fwd['xnext']).swapaxes(0, 1)
X_test_fwd['z'] = np.array(X_test_fwd['z']).swapaxes(0, 1)
X_test_hat = X_test_fwd["xnext"]
X_train_hat = X_train_fwd["xnext"]
# Unpad using the masks
#after masking, need to rehsape to (nt, nfeat)
#Compute mean absolute error over all sequences
mse_train = np.mean([
mean_absolute_error(xval, xhat)
for (xval, xhat) in zip(X_train, X_train_hat)
])
print('MSE over the train set: ' + str(mse_train))
#Compute mean absolute error over all sequences
mse_test = np.mean([
mean_absolute_error(xval, xhat)
for (xval, xhat) in zip(X_test, X_test_hat)
])
print('MSE over the test set: ' + str(mse_test))
#plot validation and
plot_total_loss(model.loss['total'], model.val_loss['total'], "Total loss",
out_dir, "total_loss.png")
plot_total_loss(model.loss['kl'], model.val_loss['kl'], "kl_loss", out_dir,
"kl_loss.png")
plot_total_loss(model.loss['ll'], model.val_loss['ll'], "ll_loss", out_dir,
"ll_loss.png") #Negative to see downard curve
# Visualization of trajectories
subj = 6
feature = 0
# For train
plot_trajectory(
X_train, X_train_hat, subj, 'all', out_dir,
f'traj_train_s_{subj}_f_all') # testing for a given subject
plot_trajectory(
X_train, X_train_hat, subj, feature, out_dir,
f'traj_train_s_{subj}_f_{feature}') # testing for a given feature
# For test
plot_trajectory(X_test, X_test_hat, subj, 'all', out_dir,
f'traj_test_s_{subj}_f_all') # testing for a given subject
plot_trajectory(
X_test, X_test_hat, subj, feature, out_dir,
f'traj_test_s_{subj}_f_{feature}') # testing for a given feature
z_train = X_train_fwd['z']
z_test = X_test_fwd['z']
z_train = [Z for (i, Z) in enumerate(z_train)]
z_test = [Z for (i, Z) in enumerate(z_test)]
z = z_train + z_test
# Dir for projections
proj_path = 'z_proj/'
if not os.path.exists(out_dir + proj_path):
os.makedirs(out_dir + proj_path)
#plot latent space
for dim0 in range(p["z_dim"]):
for dim1 in range(dim0, p["z_dim"]):
if dim0 == dim1: continue # very dirty
plot_z_time_2d(z,
p["ntp"], [dim0, dim1],
out_dir + proj_path,
out_name=f'z_d{dim0}_d{dim1}')
#Sampling
# Create first samples with only one timepoint
gen_model = SinDataGenerator(p["curves"], p["ntp"], p["noise"])
samples = gen_model.generate_n_samples(500)
X_samples = np.asarray([y[:1] for (_, y) in samples])
X_samples = torch.FloatTensor(X_samples).permute((1, 0, 2))
X_sample = model.sequence_predict(X_samples.to(DEVICE), p['ntp'])
#Get the samples
X_sample['xnext'] = np.array(X_sample['xnext']).swapaxes(0, 1)
X_sample['z'] = np.array(X_sample['z']).swapaxes(0, 1)
# plot the samples over time
plot_many_trajectories(X_sample['xnext'], 'all', p["ntp"], out_dir,
'x_samples')
# Dir for projections
sampling_path = 'z_proj_sampling/'
if not os.path.exists(out_dir + sampling_path):
os.makedirs(out_dir + sampling_path)
#plot latent space
for dim0 in range(p["z_dim"]):
for dim1 in range(dim0, p["z_dim"]):
if dim0 == dim1: continue # very dirty
plot_z_time_2d(X_sample['z'],
p["ntp"], [dim0, dim1],
out_dir + sampling_path,
out_name=f'z_d{dim0}_d{dim1}')
loss = {
"mse_train": mse_train,
"mse_test": mse_test,
"loss_total": model.loss['total'][-1],
"loss_kl": model.loss['kl'][-1],
"loss_ll": model.loss['ll'][-1]
}
return loss
def run_experiment(p, csv_path, out_dir, data_cols=[]):
"""
Function to run the experiments.
p contain all the hyperparameters needed to run the experiments
We assume that all the parameters needed are present in p!!
out_dir is the out directory
#hyperparameters
"""
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#Seed
torch.manual_seed(p["seed"])
np.random.seed(p["seed"])
#Redirect output to the out dir
# sys.stdout = open(out_dir + 'output.out', 'w')
#save parameters to the out dir
with open(out_dir + "params.txt", "w") as f:
f.write(str(p))
# DEVICE
## Decidint on device on device.
DEVICE_ID = 0
DEVICE = torch.device(
'cuda:' + str(DEVICE_ID) if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
torch.cuda.set_device(DEVICE_ID)
# LOAD DATA
#Start by not using validation data
# this is a list of values
X_train, X_test, Y_train, Y_test, mri_col = load_multimodal_data(
csv_path,
data_cols,
p["ch_type"],
train_set=0.9,
normalize=True,
return_covariates=True)
p["n_feats"] = [x[0].shape[1] for x in X_train]
X_train_list = []
mask_train_list = []
X_test_list = []
mask_test_list = []
print('Length of train/test')
print(len(X_train[0]))
print(len(X_test[0]))
#For each channel, pad, create the mask, and append
for x_ch in X_train:
X_train_tensor = [torch.FloatTensor(t) for t in x_ch]
X_train_pad = nn.utils.rnn.pad_sequence(X_train_tensor,
batch_first=False,
padding_value=np.nan)
mask_train = ~torch.isnan(X_train_pad)
mask_train_list.append(mask_train.to(DEVICE))
X_train_pad[torch.isnan(X_train_pad)] = 0
X_train_list.append(X_train_pad.to(DEVICE))
for x_ch in X_test:
X_test_tensor = [torch.FloatTensor(t) for t in x_ch]
X_test_pad = nn.utils.rnn.pad_sequence(X_test_tensor,
batch_first=False,
padding_value=np.nan)
mask_test = ~torch.isnan(X_test_pad)
mask_test_list.append(mask_test.to(DEVICE))
X_test_pad[torch.isnan(X_test_pad)] = 0
X_test_list.append(X_test_pad.to(DEVICE))
# ntp = max(X_train_list[0].shape[0], X_test_list[0].shape[0])
ntp = max(max([x.shape[0] for x in X_train_list]),
max([x.shape[0] for x in X_train_list]))
model = rnnvae_h.MCRNNVAE(p["h_size"],
p["x_hidden"],
p["x_n_layers"],
p["z_hidden"],
p["z_n_layers"],
p["enc_hidden"],
p["enc_n_layers"],
p["z_dim"],
p["dec_hidden"],
p["dec_n_layers"],
p["clip"],
p["n_epochs"],
p["batch_size"],
p["n_channels"],
p["ch_type"],
p["n_feats"],
DEVICE,
print_every=100,
phi_layers=p["phi_layers"],
sigmoid_mean=p["sig_mean"],
dropout=p["dropout"],
dropout_threshold=p["drop_th"])
model.ch_name = p["ch_names"]
optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"])
model.optimizer = optimizer
model = model.to(DEVICE)
# Fit the model
# FIT IT FOR THE NUMBER OF EPOCHS, X TIMES
ntimes = 20
for nrep in range(ntimes):
print(nrep)
model.fit(X_train_list, X_test_list, mask_train_list, mask_test_list)
#fit the model after changing the lr
if p["dropout"]:
print("Print the dropout")
print(model.dropout_comp)
### After training, save the model!
model.save(out_dir, 'model.pt')
# Predict the reconstructions from X_val and X_train
X_train_fwd = model.predict(X_train_list, mask_train_list, nt=ntp)
X_test_fwd = model.predict(X_test_list, mask_test_list, nt=ntp)
# Unpad using the masks
#plot validation and
plot_total_loss(model.loss['total'], model.val_loss['total'],
"Total loss", out_dir, "total_loss.png")
plot_total_loss(model.loss['kl'], model.val_loss['kl'], "kl_loss",
out_dir, "kl_loss.png")
plot_total_loss(model.loss['ll'], model.val_loss['ll'], "ll_loss",
out_dir, "ll_loss.png") #Negative to see downard curve
#Compute mse and reconstruction loss
#General mse and reconstruction over
# test_loss = model.recon_loss(X_test_fwd, target=X_test_pad, mask=mask_test_tensor)
train_loss = model.recon_loss(X_train_fwd,
target=X_train_list,
mask=mask_train_list)
test_loss = model.recon_loss(X_test_fwd,
target=X_test_list,
mask=mask_test_list)
print('MSE over the train set: ' + str(train_loss["mae"]))
print('Reconstruction loss over the train set: ' +
str(train_loss["rec_loss"]))
print('MSE over the test set: ' + str(test_loss["mae"]))
print('Reconstruction loss the train set: ' +
str(test_loss["rec_loss"]))
pred_results = {}
for ch_name in p["ch_names"][:3]:
pred_results[f"pred_{ch_name}_mae"] = []
rec_results = {}
for ch_name in p["ch_names"]:
rec_results[f"recon_{ch_name}_mae"] = []
results = {**pred_results, **rec_results}
######################
## Prediction of last time point
######################
i = 0
# Test data without last timepoint
# X_test_tensors do have the last timepoint
pred_ch = list(range(3))
print(pred_ch)
t_pred = 1
res = eval_prediction(model, X_test, t_pred, pred_ch, DEVICE)
for (i, ch) in enumerate(
[x for (i, x) in enumerate(p["ch_names"]) if i in pred_ch]):
loss[f'pred_{ch}_mae'].append(res[i])
############################
## Test reconstruction for each channel, using the other one
############################
# For each channel
if p["n_channels"] > 1:
for i in range(len(X_test)):
curr_name = p["ch_names"][i]
av_ch = list(range(len(X_test)))
av_ch.remove(i)
mae_rec = eval_reconstruction(model, X_test, X_test_list,
mask_test_list, av_ch, i)
# Get MAE result for that specific channel over all timepoints
results[f"recon_{curr_name}_mae"] = mae_rec
loss = {
"mae_train": train_loss["mae"],
"rec_train": train_loss["rec_loss"],
"mae_test": test_loss["mae"],
"loss_total": model.loss['total'][-1],
"loss_kl": model.loss['kl'][-1],
"loss_ll": model.loss['ll'][-1],
}
if p["dropout"]:
loss["dropout_comps"] = model.dropout_comp
loss = {**loss, **results}
print(results)
"""
# Dir for projections
proj_path = 'z_proj/'
if not os.path.exists(out_dir + proj_path):
os.makedirs(out_dir + proj_path)
# Test the new function of latent space
#NEED TO ADAPT THIS FUNCTION
qzx_train = [np.array(x) for x in X_train_fwd['qzx']]
qzx_test = [np.array(x) for x in X_test_fwd['qzx']]
#Convert to standard
#Add padding so that the mask also works here
DX_train = [[x for x in elem] for elem in Y_train["DX"]]
DX_test = [[x for x in elem] for elem in Y_test["DX"]]
#Define colors
pallete_dict = {
"CN": "#2a9e1e",
"MCI": "#bfbc1a",
"AD": "#af1f1f"
}
# Get classificator labels, for n time points
out_dir_sample = out_dir + 'zcomp_ch_dx/'
if not os.path.exists(out_dir_sample):
os.makedirs(out_dir_sample)
plot_latent_space(model, qzx_test, ntp, classificator=DX_test, pallete_dict=pallete_dict, plt_tp='all',
all_plots=True, uncertainty=False, savefig=True, out_dir=out_dir_sample + '_test', mask=mask_test_list)
plot_latent_space(model, qzx_train, ntp, classificator=DX_train, pallete_dict=pallete_dict, plt_tp='all',
all_plots=True, uncertainty=False, savefig=True, out_dir=out_dir_sample + '_train', mask=mask_train_list)
out_dir_sample_t0 = out_dir + 'zcomp_ch_dx_t0/'
if not os.path.exists(out_dir_sample_t0):
os.makedirs(out_dir_sample_t0)
plot_latent_space(model, qzx_train, ntp, classificator=DX_train, pallete_dict=pallete_dict, plt_tp=[0],
all_plots=True, uncertainty=False, savefig=True, out_dir=out_dir_sample_t0 + '_train', mask=mask_train_list)
plot_latent_space(model, qzx_test, ntp, classificator=DX_test, pallete_dict=pallete_dict, plt_tp=[0],
all_plots=True, uncertainty=False, savefig=True, out_dir=out_dir_sample_t0 + '_test', mask=mask_test_list)
# Now plot color by timepoint
out_dir_sample = out_dir + 'zcomp_ch_tp/'
if not os.path.exists(out_dir_sample):
os.makedirs(out_dir_sample)
classif_train = [[i for (i, x) in enumerate(elem)] for elem in Y_train["DX"]]
classif_test = [[i for (i, x) in enumerate(elem)] for elem in Y_test["DX"]]
pallete = sns.color_palette("viridis", ntp)
pallete_dict = {i:value for (i, value) in enumerate(pallete)}
plot_latent_space(model, qzx_train, ntp, classificator=classif_train, pallete_dict=pallete_dict, plt_tp='all',
all_plots=True, uncertainty=False, savefig=True, out_dir=out_dir_sample + '_train', mask=mask_train_list)
plot_latent_space(model, qzx_test, ntp, classificator=classif_test, pallete_dict=pallete_dict, plt_tp='all',
all_plots=True, uncertainty=False, savefig=True, out_dir=out_dir_sample + '_test', mask=mask_test_list)
"""
return loss
def run_experiment(p, csv_path, out_dir, data_cols=[]):
"""
Function to run the experiments.
p contain all the hyperparameters needed to run the experiments
We assume that all the parameters needed are present in p!!
out_dir is the out directory
#hyperparameters
"""
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#Seed
torch.manual_seed(p["seed"])
np.random.seed(p["seed"])
#Redirect output to the out dir
# sys.stdout = open(out_dir + 'output.out', 'w')
#save parameters to the out dir
with open(out_dir + "params.txt", "w") as f:
f.write(str(p))
# DEVICE
## Decidint on device on device.
DEVICE_ID = 0
DEVICE = torch.device(
'cuda:' + str(DEVICE_ID) if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
torch.cuda.set_device(DEVICE_ID)
# LOAD DATA
#Start by not using validation data
# this is a list of values
X_train, X_test, Y_train, Y_test, mri_col = load_multimodal_data(
csv_path,
data_cols,
p["ch_type"],
train_set=0.95,
normalize=True,
return_covariates=True)
p["n_feats"] = [x[0].shape[1] for x in X_train]
X_train_list = []
mask_train_list = []
print('Length of train/test')
print(len(X_train[0]))
# need to deal with ntp here
ntp = max(np.max([[len(xi) for xi in x] for x in X_train]),
np.max([[len(xi) for xi in x] for x in X_train]))
if p["long_to_bl"]:
# HERE, change bl to long and repeat the values at t0 for ntp
for i in range(len(p["ch_type"])):
if p["ch_type"][i] == 'bl':
for j in range(len(X_train[i])):
X_train[i][j] = np.array([X_train[i][j][0]] * ntp)
for j in range(len(X_test[i])):
X_test[i][j] = np.array([X_test[i][j][0]] * ntp)
# p["ch_type"][i] = 'long'
#For each channel, pad, create the mask, and append
for x_ch in X_train:
X_train_tensor = [torch.FloatTensor(t) for t in x_ch]
X_train_pad = nn.utils.rnn.pad_sequence(X_train_tensor,
batch_first=False,
padding_value=np.nan)
mask_train = ~torch.isnan(X_train_pad)
mask_train_list.append(mask_train.to(DEVICE))
X_train_pad[torch.isnan(X_train_pad)] = 0
X_train_list.append(X_train_pad.to(DEVICE))
X_test_list = []
mask_test_list = []
for x_ch in X_test:
X_test_tensor = [torch.FloatTensor(t) for t in x_ch]
X_test_pad = nn.utils.rnn.pad_sequence(X_test_tensor,
batch_first=False,
padding_value=np.nan)
mask_test = ~torch.isnan(X_test_pad)
mask_test_list.append(mask_test.to(DEVICE))
X_test_pad[torch.isnan(X_test_pad)] = 0
X_test_list.append(X_test_pad.to(DEVICE))
ntp = max(max([x.shape[0] for x in X_train_list]),
max([x.shape[0] for x in X_test_list]))
model = rnnvae_h.MCRNNVAE(p["h_size"],
p["x_hidden"],
p["x_n_layers"],
p["z_hidden"],
p["z_n_layers"],
p["enc_hidden"],
p["enc_n_layers"],
p["z_dim"],
p["dec_hidden"],
p["dec_n_layers"],
p["clip"],
p["n_epochs"],
p["batch_size"],
p["n_channels"],
p["ch_type"],
p["n_feats"],
p["c_z"],
DEVICE,
print_every=100,
phi_layers=p["phi_layers"],
sigmoid_mean=p["sig_mean"],
dropout=p["dropout"],
dropout_threshold=p["drop_th"])
model.ch_name = p["ch_names"]
optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"])
model.optimizer = optimizer
model = model.to(DEVICE)
# Fit the model
model.fit(X_train_list, X_test_list, mask_train_list, mask_test_list)
#fit the model after changing the lr
#optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"]*.1)
#model.optimizer = optimizer
#print('Refining optimization...')
#model.fit(X_train_list, X_test_list, mask_train_list, mask_test_list)
if p["dropout"]:
print("Print the dropout")
print(model.dropout_comp)
### After training, save the model!
model.save(out_dir, 'model.pt')
# Predict the reconstructions from X_val and X_train
X_train_fwd = model.predict(X_train_list, mask_train_list, nt=ntp)
# Unpad using the masks
#plot validation and
plot_total_loss(model.loss['total'], model.val_loss['total'], "Total loss",
out_dir, "total_loss.png")
plot_total_loss(model.loss['kl'], model.val_loss['kl'], "kl_loss", out_dir,
"kl_loss.png")
plot_total_loss(model.loss['ll'], model.val_loss['ll'], "ll_loss", out_dir,
"ll_loss.png") #Negative to see downard curve
#Compute mse and reconstruction loss
#General mse and reconstruction over
# test_loss = model.recon_loss(X_test_fwd, target=X_test_pad, mask=mask_test_tensor)
train_loss = model.recon_loss(X_train_fwd,
target=X_train_list,
mask=mask_train_list)
print('MSE over the train set: ' + str(train_loss["mae"]))
print('Reconstruction loss over the train set: ' +
str(train_loss["rec_loss"]))
loss = {
"mae_train": train_loss["mae"],
"rec_train": train_loss["rec_loss"],
"loss_total": model.loss['total'][-1],
"loss_kl": model.loss['kl'][-1],
"loss_ll": model.loss['ll'][-1],
}
if p["dropout"]:
loss["dropout_comps"] = model.dropout_comp
print(loss)
return loss
def run_experiment(p,
out_dir,
gen_data=True,
data_suffix=None,
output_to_file=False):
"""
Function to run the experiments.
p contain all the hyperparameters needed to run the experiments
We assume that all the parameters needed are present in p!!
out_dir is the out directory
gen_data: bool that indicates if we have to generate the data or load it from disk
data_suffix: if gen_data=True, put the suffix of the data here, which will be in data/synth_data/ by defualt
#hyperparameters
"""
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#Seed
torch.manual_seed(p["seed"])
np.random.seed(p["seed"])
#Redirect output to the out dir
if output_to_file:
sys.stdout = open(out_dir + 'output.out', 'w')
#save parameters to the out dir
with open(out_dir + "params.txt", "w") as f:
f.write(str(p))
# DEVICE
## Decidint on device on device.
DEVICE_ID = 0
DEVICE = torch.device(
'cuda:' + str(DEVICE_ID) if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
torch.cuda.set_device(DEVICE_ID)
#Tensors should have the shape
# [ntp n_ch, n_batch, n_feat]
if gen_data:
synth_dir = 'data/synth_data/'
Z_train = pickle_load(synth_dir + f"ztrain{data_suffix}")
Z_test = pickle_load(synth_dir + f"ztest{data_suffix}")
X_train = pickle_load(synth_dir + f"xtrain{data_suffix}")
X_test = pickle_load(synth_dir + f"xtest{data_suffix}")
else:
lat_gen = LatentTemporalGenerator(p["ntp"], p["noise"], p["lat_dim"],
p["n_channels"], p["n_feats"])
Z_train, X_train = lat_gen.generate_samples(p["nsamples"])
Z_test, X_test = lat_gen.generate_samples(int(p["nsamples"] * 0.2),
train=False)
# Save the data used to the output disk
to_save = [Z_train, X_train, Z_test, X_test]
filenames = [f"ztrain", f"xtrain", f"ztest", f"xtest"]
for object, file in zip(to_save, filenames):
pickle_dump(object, out_dir + file)
X_train_list = []
mask_train_list = []
#generate the data, and the mask corresponding to each channel
for x_ch in X_train[:p["n_channels"]]:
X_train_tensor = [torch.FloatTensor(t) for t in x_ch]
X_train_pad_i = nn.utils.rnn.pad_sequence(X_train_tensor,
batch_first=False,
padding_value=np.nan)
mask_train = ~torch.isnan(X_train_pad_i)
mask_train_list.append(mask_train.to(DEVICE))
X_train_pad_i[torch.isnan(X_train_pad_i)] = 0
X_train_list.append(X_train_pad_i.to(DEVICE))
X_test_list = []
mask_test_list = []
for x_ch in X_test[:p["n_channels"]]:
X_test_tensor = [torch.FloatTensor(t) for t in x_ch]
X_test_pad = nn.utils.rnn.pad_sequence(X_test_tensor,
batch_first=False,
padding_value=np.nan)
mask_test = ~torch.isnan(X_test_pad)
mask_test_list.append(mask_test.to(DEVICE))
X_test_pad[torch.isnan(X_test_pad)] = 0
X_test_list.append(X_test_pad.to(DEVICE))
#Stack along first dimension
#cant do that bc last dimension (features) could be different length
p["n_feats"] = [p["n_feats"] for _ in range(p["n_channels"])]
# Prepare model
# Define model and optimizer
model = rnnvae_s.MCRNNVAE(p["h_size"],
p["enc_hidden"],
p["enc_n_layers"],
p["z_dim"],
p["dec_hidden"],
p["dec_n_layers"],
p["clip"],
p["n_epochs"],
p["batch_size"],
p["n_channels"],
p["ch_type"],
p["n_feats"],
p["c_z"],
DEVICE,
print_every=100,
phi_layers=p["phi_layers"],
sigmoid_mean=p["sig_mean"],
dropout=p["dropout"],
dropout_threshold=p["drop_th"])
model.ch_name = p["ch_names"]
optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"])
model.optimizer = optimizer
model = model.to(DEVICE)
# Fit the model
model.fit(X_train_list, X_test_list, mask_train_list, mask_test_list)
if p["dropout"]:
print("Print the dropout")
print(model.dropout_comp)
### After training, save the model!
model.save(out_dir, 'model.pt')
# Predict the reconstructions from X_val and X_train
X_test_list = model.predict(X_test_list, mask_test_list, nt=p["ntp"])
X_train_fwd = model.predict(X_train_list, mask_train_list, nt=p["ntp"])
# Unpad using the masks
#plot validation and
plot_total_loss(model.loss['total'], model.val_loss['total'], "Total loss",
out_dir, "total_loss.png")
plot_total_loss(model.loss['kl'], model.val_loss['kl'], "kl_loss", out_dir,
"kl_loss.png")
plot_total_loss(model.loss['ll'], model.val_loss['ll'], "ll_loss", out_dir,
"ll_loss.png") #Negative to see downard curve
##Latent spasce
#Reformulate things
z_train = [np.array(x).swapaxes(0, 1) for x in X_train_fwd['z']]
# Dir for projections
proj_path = 'z_proj/'
if not os.path.exists(out_dir + proj_path):
os.makedirs(out_dir + proj_path)
#plot latent space
for ch in range(p["n_channels"]):
for dim0 in range(p["z_dim"]):
for dim1 in range(dim0, p["z_dim"]):
if dim0 == dim1: continue # very dirty
plot_z_time_2d(z_train[ch],
p["ntp"], [dim0, dim1],
out_dir + proj_path,
out_name=f'z_ch_{ch}_d{dim0}_d{dim1}')
# Dir for projections
sampling_path = 'z_proj_sampling/'
if not os.path.exists(out_dir + sampling_path):
os.makedirs(out_dir + sampling_path)
# Test the new function of latent space
qzx = [np.array(x) for x in X_train_fwd['qzx']]
# Get classificator labels, for n time points
classif = [[i] * p["nsamples"] for i in range(p["ntp"])]
classif = np.array([str(item) for elem in classif for item in elem])
print("on_classif")
print(p["ntp"])
print(len(classif))
out_dir_sample = out_dir + 'test_zspace_function/'
if not os.path.exists(out_dir_sample):
os.makedirs(out_dir_sample)
# TODO: ADAPT THIS FUNCTION TO THE
#plot_latent_space(model, qzx, p["ntp"], classificator=classif, plt_tp='all',
# all_plots=False, uncertainty=True, savefig=True, out_dir=out_dir_sample)
loss = {
"loss_total": model.loss['total'][-1],
"loss_kl": model.loss['kl'][-1],
"loss_ll": model.loss['ll'][-1]
}
return loss
def run_experiment(p, csv_path, out_dir, data_cols=[]):
"""
Function to run the experiments.
p contain all the hyperparameters needed to run the experiments
We assume that all the parameters needed are present in p!!
out_dir is the out directory
#hyperparameters
"""
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#Seed
torch.manual_seed(p["seed"])
np.random.seed(p["seed"])
#Redirect output to the out dir
# sys.stdout = open(out_dir + 'output.out', 'w')
#save parameters to the out dir
with open(out_dir + "params.txt", "w") as f:
f.write(str(p))
# DEVICE
## Decidint on device on device.
DEVICE_ID = 0
DEVICE = torch.device(
'cuda:' + str(DEVICE_ID) if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
torch.cuda.set_device(DEVICE_ID)
# Begin on the CV data
gen = load_multimodal_data_cv(csv_path,
data_cols,
p["ch_type"],
nsplit=10,
normalize=True)
# Prepare the data structures for the data
# Load the different folds
loss = {
"mae_train": [],
"rec_train": [],
"mae_test": [],
"loss_total": [],
"loss_kl": [],
"loss_ll": [],
}
pred_results = {}
for ch_name in p["ch_names"][:3]:
pred_results[f"pred_{ch_name}_mae"] = []
rec_results = {}
for ch_name in p["ch_names"]:
rec_results[f"recon_{ch_name}_mae"] = []
loss = {**loss, **pred_results, **rec_results}
# iterator marking the fold
fold_n = 0
for X_train, X_test, Y_train, Y_test, mri_col in gen:
# LOAD DATA
#Start by not using validation data
# this is a list of values
#Create output dir for the fold
out_dir_cv = out_dir + f'_fold_{fold_n}/'
if not os.path.exists(out_dir_cv):
os.makedirs(out_dir_cv)
#Redirect output to specific folder
sys.stdout = open(out_dir_cv + 'output.out', 'w')
p["n_feats"] = [x[0].shape[1] for x in X_train]
X_train_list = []
mask_train_list = []
X_test_list = []
mask_test_list = []
print('Length of train/test')
print(len(X_train[0]))
print(len(X_test[0]))
#For each channel, pad, create the mask, and append
for x_ch in X_train:
X_train_tensor = [torch.FloatTensor(t) for t in x_ch]
X_train_pad = nn.utils.rnn.pad_sequence(X_train_tensor,
batch_first=False,
padding_value=np.nan)
mask_train = ~torch.isnan(X_train_pad)
mask_train_list.append(mask_train.to(DEVICE))
X_train_pad[torch.isnan(X_train_pad)] = 0
X_train_list.append(X_train_pad.to(DEVICE))
for x_ch in X_test:
X_test_tensor = [torch.FloatTensor(t) for t in x_ch]
X_test_pad = nn.utils.rnn.pad_sequence(X_test_tensor,
batch_first=False,
padding_value=np.nan)
mask_test = ~torch.isnan(X_test_pad)
mask_test_list.append(mask_test.to(DEVICE))
X_test_pad[torch.isnan(X_test_pad)] = 0
X_test_list.append(X_test_pad.to(DEVICE))
# ntp = max(X_train_list[0].shape[0], X_test_list[0].shape[0])
ntp = max(max([x.shape[0] for x in X_train_list]),
max([x.shape[0] for x in X_train_list]))
model = rnnvae_h.MCRNNVAE(p["h_size"],
p["hidden"],
p["n_layers"],
p["hidden"],
p["n_layers"],
p["hidden"],
p["n_layers"],
p["z_dim"],
p["hidden"],
p["n_layers"],
p["clip"],
p["n_epochs"],
p["batch_size"],
p["n_channels"],
p["ch_type"],
p["n_feats"],
DEVICE,
print_every=100,
phi_layers=p["phi_layers"],
sigmoid_mean=p["sig_mean"],
dropout=p["dropout"],
dropout_threshold=p["drop_th"])
model.ch_name = p["ch_names"]
optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"])
model.optimizer = optimizer
model = model.to(DEVICE)
# Fit the model
model.fit(X_train_list, X_test_list, mask_train_list, mask_test_list)
#fit the model after changing the lr
#optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"]*.1)
#model.optimizer = optimizer
#print('Refining optimization...')
#model.fit(X_train_list, X_test_list, mask_train_list, mask_test_list)
if p["dropout"]:
print("Print the dropout")
print(model.dropout_comp)
### After training, save the model!
model.save(out_dir_cv, 'model.pt')
# Predict the reconstructions from X_val and X_train
X_train_fwd = model.predict(X_train_list, mask_train_list, nt=ntp)
X_test_fwd = model.predict(X_test_list, mask_test_list, nt=ntp)
# Unpad using the masks
#plot validation and
plot_total_loss(model.loss['total'], model.val_loss['total'],
"Total loss", out_dir_cv, "total_loss.png")
plot_total_loss(model.loss['kl'], model.val_loss['kl'], "kl_loss",
out_dir_cv, "kl_loss.png")
plot_total_loss(model.loss['ll'], model.val_loss['ll'], "ll_loss",
out_dir_cv,
"ll_loss.png") #Negative to see downard curve
#Compute mse and reconstruction loss
#General mse and reconstruction over
# test_loss = model.recon_loss(X_test_fwd, target=X_test_pad, mask=mask_test_tensor)
train_loss = model.recon_loss(X_train_fwd,
target=X_train_list,
mask=mask_train_list)
test_loss = model.recon_loss(X_test_fwd,
target=X_test_list,
mask=mask_test_list)
print('MSE over the train set: ' + str(train_loss["mae"]))
print('Reconstruction loss over the train set: ' +
str(train_loss["rec_loss"]))
print('MSE over the test set: ' + str(test_loss["mae"]))
print('Reconstruction loss the train set: ' +
str(test_loss["rec_loss"]))
######################
## Prediction of last time point
######################
X_test_list_minus = []
X_test_tensors = []
mask_test_list_minus = []
for x_ch in X_test:
X_test_tensor = [torch.FloatTensor(t[:-1, :]) for t in x_ch]
X_test_tensor_full = [torch.FloatTensor(t) for t in x_ch]
X_test_tensors.append(X_test_tensor_full)
X_test_pad = nn.utils.rnn.pad_sequence(X_test_tensor,
batch_first=False,
padding_value=np.nan)
mask_test = ~torch.isnan(X_test_pad)
mask_test_list_minus.append(mask_test.to(DEVICE))
X_test_pad[torch.isnan(X_test_pad)] = 0
X_test_list_minus.append(X_test_pad.to(DEVICE))
# Run prediction
#this is terribly programmed holy shit
X_test_fwd_minus = model.predict(X_test_list_minus,
mask_test_list_minus,
nt=ntp)
X_test_xnext = X_test_fwd_minus["xnext"]
# Test data without last timepoint
# X_test_tensors do have the last timepoint
i = 0
for (X_ch, ch) in zip(X_test[:3], p["ch_names"][:3]):
#Select a single channel
print(f'testing for {ch}')
y_true = [x[-1] for x in X_ch if len(x) > 1]
last_tp = [len(x) - 1 for x in X_ch
] # last tp is max size of original data minus one
y_pred = []
# for each subject, select last tp
j = 0
for tp in last_tp:
if tp < 1:
j += 1
continue # ignore tps with only baseline
y_pred.append(X_test_xnext[i][tp, j, :])
j += 1
#Process it to predict it
mae_tp_ch = mean_absolute_error(y_true, y_pred)
#save the result
loss[f'pred_{ch}_mae'].append(mae_tp_ch)
i += 1
############################
## Test reconstruction for each channel, using the other one
############################
# For each channel
if p["n_channels"] > 1:
for i in range(len(X_test)):
curr_name = p["ch_names"][i]
av_ch = list(range(len(X_test)))
av_ch.remove(i)
# try to reconstruct it from the other ones
ch_recon = model.predict(X_test_list,
mask_test_list,
nt=ntp,
av_ch=av_ch,
task='recon')
#for all existing timepoints
y_true = X_test[i]
# swap dims to iterate over subjects
y_pred = np.transpose(ch_recon["xnext"][i], (1, 0, 2))
y_pred = [
x_pred[:len(x_true)]
for (x_pred, x_true) in zip(y_pred, y_true)
]
#prepare it timepoint wise
y_pred = [tp for subj in y_pred for tp in subj]
y_true = [tp for subj in y_true for tp in subj]
mae_rec_ch = mean_absolute_error(y_true, y_pred)
# Get MAE result for that specific channel over all timepoints
loss[f"recon_{curr_name}_mae"].append(mae_rec_ch)
# Save results in the loss object
loss["mae_train"].append(train_loss["mae"])
loss["rec_train"].append(train_loss["rec_loss"])
loss["mae_test"].append(train_loss["mae"])
loss["loss_total"].append(model.loss['total'][-1])
loss["loss_kl"].append(model.loss['kl'][-1])
loss["loss_ll"].append(model.loss['ll'][-1])
fold_n += 1
# break at 5 iterations, need to do it faster
if fold_n == 5:
break
# Compute the mean for every param in the loss dict
for k in loss.keys():
loss[k] = np.mean(loss[k])
print(loss)
return loss
def run_experiment(p, csv_path, out_dir, data_cols=[]):
"""
Function to run the experiments.
p contain all the hyperparameters needed to run the experiments
We assume that all the parameters needed are present in p!!
out_dir is the out directory
#hyperparameters
"""
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#Seed
torch.manual_seed(p["seed"])
np.random.seed(p["seed"])
#Redirect output to the out dir
# sys.stdout = open(out_dir + 'output.out', 'w')
#save parameters to the out dir
with open(out_dir + "params.txt", "w") as f:
f.write(str(p))
# DEVICE
## Decidint on device on device.
DEVICE_ID = 0
DEVICE = torch.device(
'cuda:' + str(DEVICE_ID) if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
torch.cuda.set_device(DEVICE_ID)
# LOAD DATA
#Start by not using validation data
# this is a list of values
X_train, _, Y_train, _, mri_col = load_multimodal_data(
csv_path,
data_cols,
train_set=1.0,
normalize=True,
return_covariates=True)
p["n_feats"] = [x[0].shape[1] for x in X_train]
X_train_list = []
mask_train_list = []
#For each channel, pad, create the mask, and append
for x_ch in X_train:
X_train_tensor = [torch.FloatTensor(t) for t in x_ch]
X_train_pad = nn.utils.rnn.pad_sequence(X_train_tensor,
batch_first=False,
padding_value=np.nan)
mask_train = ~torch.isnan(X_train_pad)
mask_train_list.append(mask_train.to(DEVICE))
X_train_pad[torch.isnan(X_train_pad)] = 0
X_train_list.append(X_train_pad.to(DEVICE))
ntp = max([X_train_list[i].shape[0] for i in range(len(X_train_list))])
model = rnnvae.MCRNNVAE(p["h_size"], p["hidden"], p["n_layers"],
p["hidden"], p["n_layers"], p["hidden"],
p["n_layers"], p["z_dim"], p["hidden"],
p["n_layers"], p["clip"], p["n_epochs"],
p["batch_size"], p["n_channels"], p["n_feats"],
DEVICE)
model.ch_name = p["ch_names"]
optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"])
model.optimizer = optimizer
model = model.to(DEVICE)
# Fit the model
model.fit(X_train_list, X_train_list, mask_train_list, mask_train_list)
### After training, save the model!
model.save(out_dir, 'model.pt')
# Predict the reconstructions from X_val and X_train
X_train_fwd = model.predict(X_train_list, nt=ntp)
# Unpad using the masks
#plot validation and
plot_total_loss(model.loss['total'], model.val_loss['total'], "Total loss",
out_dir, "total_loss.png")
plot_total_loss(model.loss['kl'], model.val_loss['kl'], "kl_loss", out_dir,
"kl_loss.png")
plot_total_loss(model.loss['ll'], model.val_loss['ll'], "ll_loss", out_dir,
"ll_loss.png") #Negative to see downard curve
#Compute mse and reconstruction loss
#General mse and reconstruction over
# test_loss = model.recon_loss(X_test_fwd, target=X_test_pad, mask=mask_test_tensor)
train_loss = model.recon_loss(X_train_fwd,
target=X_train_list,
mask=mask_train_list)
print('MSE over the train set: ' + str(train_loss["mae"]))
print('Reconstruction loss over the train set: ' +
str(train_loss["rec_loss"]))
# print('MSE over the test set: ' + str(test_loss["mae"]))
# print('Reconstruction loss the train set: ' + str(test_loss["rec_loss"]))
##Latent spasce
#Reformulate things
#z_train = [np.array(x).swapaxes(0,1) for x in X_train_fwd['z']]
# IT DOESNT WORK RIGHT NOW
# Not needed rn
# z_train = []
#for (i, z_ch) in enumerate(X_train_fwd['z']):
# mask_ch = mask_train_list[i].cpu().numpy()
# z_train.append([X[np.tile(mask_ch[:,j,0], (p["z_dim"], 1)).T].reshape((-1, p["z_dim"])) for (j, X) in enumerate(z_ch)])
# X_test_hat = [X[mask_test[:,i,:]].reshape((-1, nfeatures)) for (i, X) in enumerate(X_test_hat)]
# z_test = [np.array(x).swapaxes(0,1) for x in X_test_fwd['z']]
#Zspace needs to be masked
# Dir for projections
proj_path = 'z_proj/'
if not os.path.exists(out_dir + proj_path):
os.makedirs(out_dir + proj_path)
#plot latent space for ALL the
#Por plotting the latent space, we need to do a similar function to plot_latent_space. Wait, that directly does it
#for ch in range(p["n_channels"]):
# for dim0 in range(p["z_dim"]):
# for dim1 in range(dim0, p["z_dim"]):
# if dim0 == dim1: continue # very dirty
# plot_z_time_2d(z_train[ch], ntp, [dim0, dim1], out_dir + proj_path, out_name=f'z_ch_{ch}_d{dim0}_d{dim1}')
# Test the new function of latent space
#NEED TO ADAPT THIS FUNCTION
qzx = [np.array(x) for x in X_train_fwd['qzx']]
print('len qzx')
print(len(qzx))
# Get classificator labels, for n time points
out_dir_sample = out_dir + 'zcomp_ch_dx/'
if not os.path.exists(out_dir_sample):
os.makedirs(out_dir_sample)
dx_dict = {
"NL": "CN",
"MCI": "MCI",
"MCI to NL": "CN",
"Dementia": "AD",
"Dementia to MCI": "MCI",
"NL to MCI": "MCI",
"NL to Dementia": "AD",
"MCI to Dementia": "AD"
}
#Convert to standard
#Add padding so that the mask also works here
DX = [[x for x in elem] for elem in Y_train["DX"]]
#Define colors
pallete_dict = {"CN": "#2a9e1e", "MCI": "#bfbc1a", "AD": "#af1f1f"}
plot_latent_space(model,
qzx,
ntp,
classificator=DX,
pallete_dict=pallete_dict,
plt_tp='all',
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample,
mask=mask_train_list)
out_dir_sample_t0 = out_dir + 'zcomp_ch_dx_t0/'
if not os.path.exists(out_dir_sample_t0):
os.makedirs(out_dir_sample_t0)
plot_latent_space(model,
qzx,
ntp,
classificator=DX,
pallete_dict=pallete_dict,
plt_tp=[0],
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample_t0,
mask=mask_train_list)
# Now plot color by timepoint
out_dir_sample = out_dir + 'zcomp_ch_tp/'
if not os.path.exists(out_dir_sample):
os.makedirs(out_dir_sample)
classif = [[i for (i, x) in enumerate(elem)] for elem in Y_train["DX"]]
pallete = sns.color_palette("viridis", ntp)
pallete_dict = {i: value for (i, value) in enumerate(pallete)}
plot_latent_space(model,
qzx,
ntp,
classificator=classif,
pallete_dict=pallete_dict,
plt_tp='all',
all_plots=True,
uncertainty=False,
savefig=True,
out_dir=out_dir_sample,
mask=mask_train_list)
loss = {
"mse_train": train_loss["mae"],
"rec_train": train_loss["rec_loss"],
# "mse_test": test_loss["mae"],
"loss_total": model.loss['total'][-1],
"loss_kl": model.loss['kl'][-1],
"loss_ll": model.loss['ll'][-1]
}
return loss
def run_experiment(p, csv_path, out_dir, data_cols='_mri_vol'):
"""
Function to run the experiments.
p contain all the hyperparameters needed to run the experiments
We assume that all the parameters needed are present in p!!
out_dir is the out directory
#hyperparameters
"""
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#Seed
torch.manual_seed(p["seed"])
np.random.seed(p["seed"])
#Redirect output to the out dir
# sys.stdout = open(out_dir + 'output.out', 'w')
#save parameters to the out dir
with open(out_dir + "params.txt", "w") as f:
f.write(str(p))
# DEVICE
## Decidint on device on device.
DEVICE_ID = 0
DEVICE = torch.device(
'cuda:' + str(DEVICE_ID) if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
torch.cuda.set_device(DEVICE_ID)
# LOAD DATA
X_train, X_test, Y_train, Y_test, mri_col = open_MRI_data_var(
csv_path,
train_set=0.9,
normalize=True,
return_covariates=True,
data_cols=data_cols)
#TEMPORAL
#Combine test and train Y for later
Y = {}
for k in Y_train.keys():
Y[k] = Y_train[k] + Y_test[k]
# List of (nt, nfeatures) numpy objects
p["x_size"] = X_train[0].shape[1]
print(p["x_size"])
# Apply padding to both X_train and X_val
# REMOVE LAST POINT OF EACH INDIVIDUAL
X_train_tensor = [torch.FloatTensor(t[:-1, :]) for t in X_train]
X_train_pad = nn.utils.rnn.pad_sequence(X_train_tensor,
batch_first=False,
padding_value=np.nan)
X_test_tensor = [torch.FloatTensor(t) for t in X_test]
X_test_pad = nn.utils.rnn.pad_sequence(X_test_tensor,
batch_first=False,
padding_value=np.nan)
p["ntp"] = max(X_train_pad.size(0), X_test_pad.size(0))
# Those datasets are of size [Tmax, Batch_size, nfeatures]
# Save mask to unpad later when testing
mask_train = ~torch.isnan(X_train_pad)
mask_test = ~torch.isnan(X_test_pad)
# convert to tensor
mask_train_tensor = torch.BoolTensor(mask_train)
mask_test_tensor = torch.BoolTensor(mask_test)
#convert those NaN to zeros
X_train_pad[torch.isnan(X_train_pad)] = 0
X_test_pad[torch.isnan(X_test_pad)] = 0
# Define model and optimizer
model = rnnvae.ModelRNNVAE(p["x_size"], p["h_size"], p["hidden"],
p["n_layers"], p["hidden"], p["n_layers"],
p["hidden"], p["n_layers"], p["z_dim"],
p["hidden"], p["n_layers"], p["clip"],
p["n_epochs"], p["batch_size"], DEVICE)
optimizer = torch.optim.Adam(model.parameters(), lr=p["learning_rate"])
model.optimizer = optimizer
model = model.to(DEVICE)
# Fit the model
model.fit(X_train_pad.to(DEVICE), X_test_pad.to(DEVICE),
mask_train_tensor.to(DEVICE), mask_test_tensor.to(DEVICE))
### After training, save the model!
model.save(out_dir, 'model.pt')
# Predict the reconstructions from X_val and X_train
X_test_fwd = model.predict(X_test_pad.to(DEVICE))
X_train_fwd = model.predict(X_train_pad.to(DEVICE))
#Reformulate things
X_train_fwd['xnext'] = np.array(X_train_fwd['xnext']).swapaxes(0, 1)
X_train_fwd['z'] = np.array(X_train_fwd['z']).swapaxes(0, 1)
X_test_fwd['xnext'] = np.array(X_test_fwd['xnext']).swapaxes(0, 1)
X_test_fwd['z'] = np.array(X_test_fwd['z']).swapaxes(0, 1)
X_test_hat = X_test_fwd["xnext"]
X_train_hat = X_train_fwd["xnext"]
# Unpad using the masks
#after masking, need to rehsape to (nt, nfeat)
X_test_hat = [
X[mask_test[:, i, :]].reshape((-1, p["x_size"]))
for (i, X) in enumerate(X_test_hat)
]
X_train_hat = [
X[mask_train[:, i, :]].reshape((-1, p["x_size"]))
for (i, X) in enumerate(X_train_hat)
]
#Compute mean absolute error over all sequences
mse_train = np.mean([
mean_absolute_error(xval[:-1, :], xhat)
for (xval, xhat) in zip(X_train, X_train_hat)
])
print('MSE over the train set: ' + str(mse_train))
#Compute mean absolute error over all sequences
mse_test = np.mean([
mean_absolute_error(xval, xhat)
for (xval, xhat) in zip(X_test, X_test_hat)
])
print('MSE over the test set: ' + str(mse_test))
#plot validation and
plot_total_loss(model.loss['total'], model.val_loss['total'], "Total loss",
out_dir, "total_loss.png")
plot_total_loss(model.loss['kl'], model.val_loss['kl'], "kl_loss", out_dir,
"kl_loss.png")
plot_total_loss(model.loss['ll'], model.val_loss['ll'], "ll_loss", out_dir,
"ll_loss.png") #Negative to see downard curve
# Visualization of trajectories
"""
subj = 6
feature = 12
# For train
plot_trajectory(X_train, X_train_hat, subj, 'all', out_dir, f'traj_train_s_{subj}_f_all') # testing for a given subject
plot_trajectory(X_train, X_train_hat, subj, feature, out_dir, f'traj_train_s_{subj}_f_{feature}') # testing for a given feature
# For test
plot_trajectory(X_test, X_test_hat, subj, 'all', out_dir, f'traj_test_s_{subj}_f_all') # testing for a given subject
plot_trajectory(X_test, X_test_hat, subj, feature, out_dir, f'traj_test_s_{subj}_f_{feature}') # testing for a given feature
"""
z_train = X_train_fwd['z']
z_test = X_test_fwd['z']
# select only the existing time points
# Repeat the mask for each latent features, as we can have variable features, need to treat the mask
#Use ptile to repeat it as many times as p["z_dim"], and transpose it
z_test = [
X[np.tile(mask_test[:, i, 0], (p["z_dim"], 1)).T].reshape(
(-1, p["z_dim"])) for (i, X) in enumerate(z_test)
]
z_train = [
X[np.tile(mask_train[:, i, 0], (p["z_dim"], 1)).T].reshape(
(-1, p["z_dim"])) for (i, X) in enumerate(z_train)
]
z = z_train + z_test
# Dir for projections
proj_path = 'z_proj/'
if not os.path.exists(out_dir + proj_path):
os.makedirs(out_dir + proj_path)
#plot latent space
for dim0 in range(p["z_dim"]):
for dim1 in range(dim0, p["z_dim"]):
if dim0 == dim1: continue # very dirty
plot_z_time_2d(z,
p["ntp"], [dim0, dim1],
out_dir + proj_path,
out_name=f'z_d{dim0}_d{dim1}')
# Dir for projections
sampling_path = 'z_proj_dx/'
if not os.path.exists(out_dir + sampling_path):
os.makedirs(out_dir + sampling_path)
#plot latent space
for dim0 in range(p["z_dim"]):
for dim1 in range(dim0, p["z_dim"]):
if dim0 == dim1: continue # very dirty
plot_z_time_2d(z,
p["ntp"], [dim0, dim1],
out_dir + sampling_path,
c='DX',
Y=Y,
out_name=f'z_d{dim0}_d{dim1}')
# Dir for projections
sampling_path = 'z_proj_age/'
if not os.path.exists(out_dir + sampling_path):
os.makedirs(out_dir + sampling_path)
#plot latent space
for dim0 in range(p["z_dim"]):
for dim1 in range(dim0, p["z_dim"]):
if dim0 == dim1: continue # very dirty
plot_z_time_2d(z,
p["ntp"], [dim0, dim1],
out_dir + sampling_path,
c='AGE',
Y=Y,
out_name=f'z_d{dim0}_d{dim1}')
# Compute MSE
# Predict for max+1 and select only the positions that I am interested in
#this sequence predict DO NOT work well
Y_true = [p[-1, :] for p in X_train]
Y_pred = []
for i in range(X_train_pad.size(1)):
x = torch.FloatTensor(X_train[i][:-1, :])
x = x.unsqueeze(1)
tp = x.size(0) # max time points (and timepoint to predict)
if tp == 0:
continue
X_fwd = model.sequence_predict(x.to(DEVICE), tp + 1)
X_hat = X_fwd['xnext']
Y_pred.append(X_hat[tp, 0, :]) #get predicted point
#For each patient in X_hat, saveonly the timepoint that we want
#Compute mse
mse_predict = mean_squared_error(Y_true, Y_pred)
print('MSE over a future timepoint prediction: ' + str(mse_predict))
# TODO: THIS SAMPLING PROCEDURE NEEDS TO BE UPDATED
"""
nt = len(X_train_pad)
nsamples = 1000
X_sample = model.sample_latent(nsamples, nt)
#Get the samples
X_sample['xnext'] = np.array(X_sample['xnext']).swapaxes(0,1)
X_sample['z'] = np.array(X_sample['z']).swapaxes(0,1)
# Dir for projections
sampling_path = 'z_proj_sampling/'
if not os.path.exists(out_dir + sampling_path):
os.makedirs(out_dir + sampling_path)
#plot latent space
for dim0 in range(p["z_dim"]):
for dim1 in range(dim0, p["z_dim"]):
if dim0 == dim1: continue # very dirty
plot_z_time_2d(X_sample['z'], p["ntp"], [dim0, dim1], out_dir + 'z_proj_sampling/', out_name=f'z_d{dim0}_d{dim1}')
"""
loss = {
"mse_train": mse_train,
"mse_test": mse_test,
"mse_predict": mse_predict,
"loss_total": model.loss['total'][-1],
"loss_kl": model.loss['kl'][-1],
"loss_ll": model.loss['ll'][-1]
}
return loss