def load_existing_weights_if_exist(resdir,
model,
model_name='model',
log=None,
device='cuda',
index_mod=-1,
res_model_file=None):
from utils_file import gfile, get_parent_path
ep_start = 0
if res_model_file is None:
resume_mod = gfile(resdir, '.*pt$')
else:
resume_mod = [res_model_file]
if len(resume_mod) > 0:
dir_mod, fn = get_parent_path(resume_mod)
ffn = [ff[ff.find('_ep') + 3:-3] for ff in fn]
key_list = []
for fff, fffn in zip(ffn, fn):
if '_it' in fff:
ind = fff.find('_it')
ep = int(fff[0:ind])
it = int(fff[ind + 3:])
else:
ep = int(fff)
it = 100000000
key_list.append([fffn, ep, it])
aa = np.array(sorted(key_list, key=lambda x: (x[1], x[2])))
name_sorted, ep_sorted = aa[:, 0], aa[:, 1]
ep_start = int(ep_sorted[index_mod])
thelast = dir_mod[0] + '/' + name_sorted[index_mod]
log.info('RESUME model from epoch {} weight loaded from {}'.format(
ep_start, thelast))
tl = torch.load(thelast, map_location=device)
if model_name not in tl:
model_name = list(tl.items())[0][0]
prefix = 'model.'
state_dict = tl[model_name]
aa = next(iter(state_dict))
if prefix in aa:
new_state_dict = OrderedDict()
for k, v in state_dict.items():
name = k[len(prefix):] # remove 'module.' of dataparallel
new_state_dict[name] = v
model.load_state_dict(new_state_dict)
else:
model.load_state_dict(tl[model_name])
else:
log.info('New training starting epoch {}'.format(ep_start))
thelast = resdir
log.info('Resdir is {}'.format(resdir))
return ep_start, get_parent_path([thelast])[1][0]
def set_model_from_file(self, file_name, cuda):
resdir_name = get_parent_path(file_name, 2)[1]
print('loading {} \nfrom dir {}'.format(
get_parent_path(file_name)[1], resdir_name))
if 'ConvN_C16_256_Lin40_50' in resdir_name:
conv_block, linear_block = [16, 32, 64, 128, 256], [40, 50]
network_name = 'ConvN'
else:
raise ('did not recognise model type')
if 'L1' in resdir_name:
losstype = 'L1'
elif 'MSE' in resdir_name:
losstype = 'MSE'
if '_Size182' in resdir_name:
in_size = [182, 218, 182]
batch_norm = True if '_BN_' in resdir_name else False
ind_drop = resdir_name.find('_D')
substr = resdir_name[ind_drop + 2:]
ii = substr.find('_')
dropout = float(substr[:ii])
drop_conv = 0
if '_DC' in resdir_name:
ind_drop = resdir_name.find('_DC')
substr = resdir_name[ind_drop + 3:]
ii = substr.find('_')
drop_conv = float(substr[:ii])
ind_lr = resdir_name.find('_lr')
lr = float(resdir_name[ind_lr + 3:])
par_model = {
'network_name': network_name,
'losstype': losstype,
'lr': lr,
'conv_block': conv_block,
'linear_block': linear_block,
'dropout': dropout,
'drop_conv': drop_conv,
'batch_norm': batch_norm,
'in_size': in_size,
'cuda': cuda,
'max_epochs': 1
}
self.set_model(par_model,
res_model_file=file_name,
verbose=False,
log_filename='eval.log')
def __getitem__(self, index: int) -> Subject:
if not isinstance(index, int):
raise ValueError(f'Index "{index}" must be int, not {type(index)}')
if self.load_from_dir:
subject = torch.load(self._subjects[index])
if self.add_to_load is not None:
#print('adding subject with {}'.format(self.add_to_load))
ii = subject.get_images()
image_path = ii[0]['path']
if 'original' in self.add_to_load:
#print('adding original subject')
ss = Subject(image = Image(image_path, INTENSITY))
# sss = self._get_sample_dict_from_subject(ss)
#sss = copy.deepcopy(ss)
sss = ss
subject['original'] = sss['image']
if self.add_to_load=='original': #trick to use both orig and mask :hmmm....
add_to_load = None
else:
add_to_load = self.add_to_load[8:]
else:
add_to_load = self.add_to_load
if add_to_load is not None:
image_add = gfile(get_parent_path(image_path), self.add_to_load_regexp)[0]
#print('adding image {} to {}'.format(image_add,self.add_to_load))
ss = Subject(image = Image(image_add, LABEL))
#sss = self._get_sample_dict_from_subject(ss)
#sss = copy.deepcopy(ss)
sss = ss
hh = subject.history
for hhh in hh:
if 'RandomElasticDeformation' in hhh[0]:
from torchio.transforms import RandomElasticDeformation
num_cp = hhh[1]['coarse_grid'].shape[1]
rr = RandomElasticDeformation(num_control_points=num_cp)
sss = rr.apply_given_transform(sss, hhh[1]['coarse_grid'])
subject[add_to_load] = sss['image']
#print('subject with keys {}'.format(subject.keys()))
else:
subject = self._subjects[index]
subject = copy.deepcopy(subject) # cheap since images not loaded yet
if self.load_getitem:
subject.load()
# Apply transform (this is usually the bottleneck)
if self._transform is not None:
subject = self._transform(subject)
if self.save_to_dir is not None:
res_dir = self.save_to_dir
fname = res_dir + '/subject{:05d}'.format(index)
if 'image_orig' in subject: subject.pop('image_orig')
torch.save(subject, fname + '_subject.pt')
return subject
def write_cati_csv():
import pandas as pd
data_path = '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/CATI_datasets/'
fcsv = data_path + 'all_cati.csv'
res = pd.read_csv(fcsv)
ser_dir = res.cenir_QC_path
ser_dir = res.cenir_QC_path[res.globalQualitative > 3].values
dcat = gdir(ser_dir, 'cat12')
fT1 = gfile(dcat, '^s.*nii')
fms = gfile(dcat, '^ms.*nii')
fs_brain = gfile(dcat, '^brain_s.*nii')
# return fT1, fms, fs_brain
ind_perm = np.random.permutation(range(0, len(fT1)))
itrain = ind_perm[0:100]
ival = ind_perm[100:]
dd = pd.DataFrame({'filename': fT1})
dd.to_csv(data_path + 'cati_cenir_QC4_all_T1.csv', index=False)
dd.loc[ival, :].to_csv(data_path + 'cati_cenir_QC4_val_T1.csv',
index=False)
dd.loc[itrain, :].to_csv(data_path + 'cati_cenir_QC4_train_T1.csv',
index=False)
dd = pd.DataFrame({'filename': fms})
dd.to_csv(data_path + 'cati_cenir_QC4_all_ms.csv', index=False)
dd.loc[ival, :].to_csv(data_path + 'cati_cenir_QC4_val_ms.csv',
index=False)
dd.loc[itrain, :].to_csv(data_path + 'cati_cenir_QC4_train_ms.csv',
index=False)
dd = pd.DataFrame({'filename': fs_brain})
dd.to_csv(data_path + 'cati_cenir_QC4_all_brain.csv', index=False)
dd.loc[ival, :].to_csv(data_path + 'cati_cenir_QC4_val_brain.csv',
index=False)
dd.loc[itrain, :].to_csv(data_path + 'cati_cenir_QC4_train_brain.csv',
index=False)
dd = pd.DataFrame({'filename': fT1})
dd.to_csv(data_path + 'cati_cenir_all_T1.csv', index=False)
dd = pd.DataFrame({'filename': fms})
dd.to_csv(data_path + 'cati_cenir_all_ms.csv', index=False)
dd = pd.DataFrame({'filename': fs_brain})
dd.to_csv(data_path + 'cati_cenir_all_brain.csv', index=False)
#add brain mask in csv
allcsv = gfile('/home/romain.valabregue/datal/QCcnn/CATI_datasets',
'cati_cenir.*csv')
for onecsv in allcsv:
res = pd.read_csv(onecsv)
resout = onecsv[:-4] + '_mask.csv'
fmask = []
for ft1 in res.filename:
d = get_parent_path(ft1)[0]
fmask += gfile(d, '^mask', opts={"items": 1})
res['brain_mask'] = fmask
res.to_csv(resout, index=False)
def get_train_and_val_csv(names='', root_fs='lustre'):
return_list = True
if isinstance(names, str):
names = [names]
return_list = False
print('name is {}'.format(type(names)))
if root_fs == 'lustre':
data_path_hcp = '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/'
elif root_fs == 'le70':
data_path_hcp = '/data/romain/HCPdata/'
data_path_cati = '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/CATI_datasets/'
fcsv_train, fcsv_val = [], []
for name in names:
if 'hcp' in name:
if 'T1' in name:
fname_train, fname_val = 'Motion_T1_train_hcp400.csv', 'Motion_T1_val_hcp200.csv'
elif 'brain_ms' in name:
fname_train, fname_val = 'healthy_brain_ms_train_hcp400.csv', 'healthy_brain_ms_val_hcp200.csv'
elif 'ms' in name:
fname_train, fname_val = 'healthy_ms_train_hcp400.csv', 'healthy_ms_val_hcp200.csv'
else:
print('can not guess which DATA from {}'.format(name))
raise
file_train = data_path_hcp + fname_train
file_val = data_path_hcp + fname_val
elif 'cati' in name:
if 'T1' in name:
fname_train, fname_val = 'cati_cenir_QC4_train_T1.csv', 'cati_cenir_QC4_val_T1.csv'
elif 'brain' in name:
fname_train, fname_val = 'cati_cenir_QC4_train_brain.csv', 'cati_cenir_QC4_val_brain.csv'
elif 'i_ms' in name:
fname_train, fname_val = 'cati_cenir_QC4_train_ms.csv', 'cati_cenir_QC4_val_ms.csv'
else:
print('can not guess which DATA from {}'.format(name))
raise
file_train = data_path_cati + fname_train
file_val = data_path_cati + fname_val
else:
print('can not guess which DATA from {}'.format(name))
raise
print('data {}\nfor {} \t found {} {} '.format(
get_parent_path([file_train])[0][0], name, fname_train, fname_val))
fcsv_train.append(file_train)
fcsv_val.append(file_val)
if return_list:
return fcsv_train, fcsv_val
else:
return fcsv_train[0], fcsv_val[0]
def get_subject_list_from_file_list(fin, mask_regex=None, mask_key='brain'):
subjects_list = []
for ff in fin:
one_suj = {'image': Image(ff, INTENSITY)}
if mask_regex is not None:
dir_file = get_parent_path(ff)[0]
fmask = gfile(dir_file, mask_regex, {"items": 1})
one_suj[mask_key] = Image(fmask[0], LABEL)
subjects_list.append(Subject(one_suj))
return subjects_list
parser.add_option("-w", "--saved_model", action="store", dest="saved_model", default='',
help="full path of the model's weights file ")
parser.add_option("--use_gpu", action="store", dest="use_gpu", default=0, type="int",
help="0 means no gpu 1 to 4 means gpu device (0) ")
(options, args) = parser.parse_args()
log = get_log_file()
name, val_number = options.out_name, options.val_number
saved_model = options.saved_model
cuda = True if options.use_gpu > 0 else False
if val_number<0:
out_name = name
subdir = None #'eval_rrr__{}_{}'.format(name)
else:
out_name = 'eval_num_{:04d}'.format(val_number)
subdir = 'eval_{}_{}'.format(name, get_parent_path(saved_model)[1][:-3])
doit, name_suffix, target = get_dataset_from_option(options)
out_name += name_suffix
doit.set_model_from_file(saved_model, cuda=cuda)
doit.validation_droupout = True
doit.eval_regress_motion(999, 99, basename=out_name, subdir=subdir, target=target)
def get_sujname_from_label_filename(ligne, col_name):
cell_content = ligne[col_name]
array_path = eval(cell_content)
return get_parent_path(str(array_path[0]),level=4)[1]
res = gdir('/home/romain.valabregue/datal/PVsynth/training/RES_14mm_tissue','data')
res = gdir('/home/romain.valabregue/datal/PVsynth/jzay/training/RES1mm_prob','pve_synth_mod3_P128$')
res = gdir('/home/romain.valabregue/datal/PVsynth/jzay/training/RES1mm_prob','aniso')
res = gdir(res,'results_cluster')
res = ['/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/PVsynth/jzay/training/RES1mm_prob/pve_synth_mod3_P128_aniso_LogLkd_reg_multi/result',
'/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/PVsynth/jzay/training/RES1mm_prob/pve_synth_mod3_P128_aniso_LogLkd_reg_unis_lam1/results_cluster',
'/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/PVsynth/jzay/training/RES1mm_prob/pve_synth_mod3_P128_aniso_LogLkd_classif/results_cluster',
]
res = ['/home/romain.valabregue/datal/PVsynth/jzay/training/RES1mm_prob/pve_synth_mod3_P128/results_cluster/']
report_learning_curves(res)
#explore synthetic data histogram
results_dirs = glob.glob('/home/romain.valabregue/datal/PVsynth/RES_1.4mm/t*/513130')
f=gfile(results_dirs,'^5.*nii')
resname = get_parent_path(results_dirs,2)[1]
resfig = '/home/romain.valabregue/datal/PVsynth/figure/volume_synth/'
for i, ff in enumerate(f):
img = nb.load(ff)
data = img.get_fdata(dtype=np.float32)
fig = plt.figure(resname[i])
hh = plt.hist(data.flatten(), bins=500)
axes = plt.gca()
axes.set_ylim([0 , 40000])
#fig.savefig(resfig + resname[i] + '.png')
#concat eval.csv
res = '/home/romain.valabregue/datal/PVsynth/eval_cnn/RES_14mm_tissue/dataS*/*/eval.csv'
def report_learning_curves(results_dirs, save=False):
"""
Plot error curves from record files and save plot to jpeg file.
"""
def plot_losses(pattern, color, label):
files = glob.glob(results_dir + pattern)
files.sort(key=os.path.getmtime)
losses, ymean, qt05, qt95 = [], [], [], []
nb_iter_list = []
nb_iter_mem = -1
for file in files:
df = pd.read_csv(file, index_col=0)
loss = df['loss'].values
ymean.append(np.mean(loss))
qt05.append(np.quantile(loss, 0.05))
qt95.append(np.quantile(loss, 0.95))
#losses.append(loss)
nb_iter = df.shape[0]
if nb_iter_mem >= 0:
if not(nb_iter_mem == nb_iter):
print('\t {} \t {} iter previous {} '.format(os.path.basename(file), nb_iter, nb_iter_mem,))
nb_iter_mem = nb_iter
nb_iter_list.append(nb_iter)
#plt.plot(np.mean(losses, axis=1), color=color, label=label)
#plt.plot(np.quantile(losses, 0.95, axis=1), '--', color=color)
#plt.plot(np.quantile(losses, 0.05, axis=1), '--', color=color)
plt.plot(ymean, color=color, label=label)
plt.plot(qt95, '--', color=color)
plt.plot(qt05, '--', color=color)
return [np.max(nb_iter_list), np.min(nb_iter_list), ymean]
if isinstance(results_dirs,str):
results_dirs = [results_dirs]
train_loss, resname_list = [], []
for results_dir in results_dirs:
resname = get_parent_path(results_dir, level=2)[1]
print(resname)
resname_list.append(resname)
plt.figure(resname)
iter_max, iter_min, ymeans = plot_losses('/Train_ep*.csv', 'blue', 'train mean loss')
train_loss.append(ymeans)
#plot_losses('/Val_ep*.csv', 'green', 'val mean loss')
if iter_max==iter_min:
plt.xlabel('epch ( {} iter)'.format(iter_max))
else:
plt.xlabel('epoch ( {} iter {})'.format(iter_max, iter_min))
plt.legend()
plt.title('Training and validation error curves')
plt.show()
if save:
plt.savefig(results_dir + '/{}loss_curves.jpeg'.format(resname))
plt.figure('all')
for tt in train_loss:
plt.plot(tt)
plt.legend(resname_list)
def plot_train_val_results(dres, train_csv_regex='Train.*csv', val_csv_regex='Val.*csv',
prediction_column_name='prediction', target_column_name='targets',
target_scale=1, fign='Res', sresname=None):
legend_str=[]
col = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'w']
for ii, oneres in enumerate(dres):
fresT = gfile(oneres, train_csv_regex)
fresV=gfile(oneres, val_csv_regex)
if len(fresV)==0:
print('{} empty dir {} '.format(colored('Skiping','red'), get_parent_path(oneres)[1]))
continue
is_train = False if len(fresT)==0 else True
if is_train: resT = [pd.read_csv(ff) for ff in fresT]
resdir, resname = get_parent_path(fresV)
nbite = len(resT[0]) if is_train else 80000
fresV_sorted, b, c = get_ep_iter_from_res_name(resname, nbite)
ite_tot = c+b*nbite
ite_tottt = np.hstack([0, ite_tot])
print(ite_tot)
resV = [pd.read_csv(resdir[0] + '/' + ff) for ff in fresV_sorted]
df_val = pd.concat(resV, ignore_index=True, sort=False)
for rr in resV:
if 'sample_time' not in rr:
rr['sample_time'] = rr['batch_time'] / 4 #for old result always runed with batchsize 4
if isinstance(rr[prediction_column_name][0], str):
rr[prediction_column_name] = rr[prediction_column_name].apply(
lambda s: convert_string_array_to_array(s))
rr[target_column_name] = rr[target_column_name].apply(
lambda s: convert_string_array_to_array(s))
if is_train:
for rr in resT:
if 'sample_time' not in rr:
rr['sample_time'] = rr['batch_time'] / 4
if isinstance(rr[prediction_column_name][0], str):
rr[prediction_column_name] = rr[prediction_column_name].apply(
lambda s: convert_string_array_to_array(s))
rr[target_column_name] = rr[target_column_name].apply(
lambda s: convert_string_array_to_array(s))
if is_train:
df_train = pd.concat(resT, ignore_index=True, sort=False)
errorT = np.abs(df_train.loc[:,prediction_column_name].values -df_train.loc[:, target_column_name].values*target_scale)
train_time = df_train.loc[:,'sample_time']
#average between validation point itte_tot
LmTrain = [np.mean(errorT[ite_tottt[ii]:ite_tottt[ii+1]]) for ii in range(0, len(ite_tot)) ]
TimeTrain = [np.mean(train_time[ite_tottt[ii]:ite_tottt[ii + 1]]) for ii in range(0, len(ite_tot))]
LmVal = [np.mean(np.abs(rr.loc[:,prediction_column_name]-rr.loc[:, target_column_name].values*target_scale)) for rr in resV]
#LmVal = np.mean(np.abs(df_val.loc[:,prediction_column_name].values - df_val.loc[:,target_column_name].values*target_scale))
TimeVal = [np.mean(rr.loc[:,'sample_time']) for rr in resV]
plt.figure('MeanL1_'+fign); legend_str.append('V{}'.format(sresname[ii]));
if is_train: legend_str.append('T{}'.format(sresname[ii]))
plt.plot(ite_tot, LmVal,'--',color=col[ii])
if is_train: plt.plot(ite_tot, LmTrain,color=col[ii], linewidth=6)
plt.figure('Time_'+fign);
plt.plot(ite_tot, TimeVal,'--',color=col[ii])
if is_train: plt.plot(ite_tot, TimeTrain,color=col[ii], linewidth=6)
#print some summary information on the results
if not is_train:
TimeTrain=0
nb_res = len(resT) if is_train else 0
np_iter = len(resT[0]) if is_train else 0
totiter, mbtt, mbtv = ite_tot[-1] / 1000, np.nanmean(TimeTrain), np.mean(TimeVal)
tot_time = nb_res * np_iter * mbtt + len(resV) * len(resV[0]) * mbtv
percent_train = nb_res * np_iter * mbtt / tot_time
tot_time_day = np.floor( tot_time/60/60/24 )
tot_time_hour = (tot_time - tot_time_day*24*60*60) / 60/60
print('Result : {} \t {} '.format(
colored(get_parent_path(resdir[0])[1], 'green'), sresname[ii] ))
print('\t{} epoch of {} vol {} val on {} vol Tot ({:.1f}%train) {} d {:.1f} h'.format(
nb_res, np_iter, len(resV),len(resV[0]), percent_train, tot_time_day, tot_time_hour ))
fj = gfile(oneres,'data.json')
if len(fj)==1:
data_struc = cc.read_json(fj[0])
bs, nw = data_struc['batch_size'], data_struc['num_workers']
else:
bs, nw = 0, -1
print('\tBatch size {} \tNum worker {} \t{:.1f} mille iter \t train/val meanTime {:.2f} / {:.2f} '.format\
(bs, nw, totiter, mbtt, mbtv))
plt.figure('MeanL1_'+fign);
plt.legend(legend_str); plt.grid()
ff=plt.gcf();ff.set_size_inches([15, 7]); #ff.tight_layout()
plt.subplots_adjust(left=0.05, right=1, bottom=0.05, top=1, wspace=0, hspace=0)
plt.ylabel('L1 loss')
plt.figure('Time_'+fign);
plt.legend(legend_str); plt.grid()
plt.ylabel('time in second')
def get_pandadf_from_res_valOn_csv(dres, resname, csv_regex='res_valOn', data_name_list=None,
select_last=None, target='ssim', target_scale=1):
if len(dres) != len(resname) : raise('length problem between dres and resname')
resdf_list = []
for oneres, resn in zip(dres, resname):
fres_valOn = gfile(oneres, csv_regex)
print('Found {} <{}> for {} '.format(len(fres_valOn), csv_regex, resn))
if len(fres_valOn) == 0:
continue
ftrain = gfile(oneres, 'res_train_ep01.csv')
rrt = pd.read_csv(ftrain[0])
nb_it = rrt.shape[0];
resdir, resname_val = get_parent_path(fres_valOn)
resname_sorted, b, c = get_ep_iter_from_res_name(resname_val, 0)
if select_last is not None:
if select_last<0:
resname_sorted = resname_sorted[select_last:]
else:
nb_iter = b*nb_it+c
resname_sorted = resname_sorted[np.argwhere(nb_iter > select_last)[1:8]]
resV = [pd.read_csv(resdir[0] + '/' + ff) for ff in resname_sorted]
resdf = pd.DataFrame()
for ii, fres in enumerate(resname_sorted):
iind = [i for i, s in enumerate(data_name_list) if s in fres]
if len(iind) ==1: #!= 1: raise ("bad size do not find which sample")
data_name = data_name_list[iind[0]]
else:
data_name = 'res_valds'
iind = fres.find(data_name)
ddn = remove_extension(fres[iind + len(data_name) + 1:])
new_col_name = 'Mout_' + ddn
iind = ddn.find('model_ep')
if iind==0:
transfo='raw'
else:
transfo = ddn[:iind - 1]
if transfo[0] == '_': #if start with _ no legend ... !
transfo = transfo[1:]
model_name = ddn[iind:]
aa, bb, cc = get_ep_iter_from_res_name([fres], nb_it)
nb_iter = bb[0] * nb_it + cc[0]
rr = resV[ii].copy()
rr['evalOn'], rr['transfo'] = data_name, transfo
rr['model_name'], rr['submodel_name'], rr['nb_iter'] = resn, model_name, str(nb_iter)
rr[target] = rr[target] * target_scale
resdf = pd.concat([resdf, rr], axis=0, sort=True)
resdf['error'] = resdf[target] - resdf['model_out']
resdf['error_abs'] = np.abs(resdf[target] - resdf['model_out'])
resdf_list.append(resdf)
return resdf_list
from scipy.linalg import logm, expm
weights, matrices = ss[0], affs
logs = [w * logm(A) for (w, A) in zip(weights, matrices)]
logs = np.array(logs)
logs_sum = logs.sum(axis=0)
expm(logs_sum/np.sum(weights, axis=0) )
#a 10-2 pres c'est bien l'identite !
rp_files = gfile('/data/romain/HCPdata/suj_274542/Motion_ms','^rp')
rp_files = gfile('/data/romain/HCPdata/suj_274542/mot_separate','^rp')
rpf = rp_files[10]
res = pd.DataFrame()
for rpf in rp_files:
dirpath,name = get_parent_path([rpf])
fout = dirpath[0] + '/check/'+name[0][3:-4] + '.nii'
t = RandomMotionFromTimeCourse(fitpars=rpf, nufft=True, oversampling_pct=0, keep_original=True, verbose=True)
dataset = ImagesDataset(suj, transform=t)
sample = dataset[0]
dicm = sample['T1']['metrics']
dicm['fname'] = fout
res = res.append(dicm, ignore_index=True)
dataset.save_sample(sample, dict(T1=fout))
fit_pars = sample['T1']['fit_pars']
plt.figure; plt.plot(fit_pars[3:].T)
plt.figure; plt.plot(fit_pars.T)
filename = res+k+'.json'
with open(filename, 'w') as file:
json.dump(conf_all[k], file, indent=4, sort_keys=True)
filename = res+'grid_search.json'
with open(filename, 'w') as file:
json.dump(gs, file, indent=4, sort_keys=True)
# generting jobs for validation
from utils_file import gdir, gfile, get_parent_path
f = gfile('/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/PVsynth/training/RES_1mm_tissue/pve_synth_data_92_common_noise_no_gamma/results_cluster',
'model.*tar')
d='/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/PVsynth/training/RES_1mm_tissue/'
dres = gdir(d,['.*','result'])
dresname = get_parent_path(dres,level=2)[1]
dresname = [dd.split('_')[0] + '_' + dd.split('_')[1] for dd in dresname]
for one_res, resn in zip(dres, dresname):
f = gfile(one_res,'model.*tar')
fname = get_parent_path(f)[1]
for ff in f:
print('\"{}\",'.format(ff))
for one_res, resn in zip(dres, dresname):
f = gfile(one_res,'model.*tar')
fname = get_parent_path(f)[1]
for ff in fname:
fname_ep = ff.split('_')[1]
def set_model(self,
par_model,
res_model_file=None,
verbose=True,
log_filename='training.log'):
network_name = par_model['network_name']
losstype = par_model['losstype']
lr = par_model['lr']
in_size = par_model['in_size']
self.cuda = par_model['cuda']
self.max_epochs = par_model['max_epochs']
optim_name = par_model['optim'] if 'optim' in par_model else 'Adam'
self.validation_droupout = par_model[
'validation_droupout'] if 'validation_droupout' in par_model else False
if network_name == 'unet_f':
self.model = UNet(
in_channels=1,
dimensions=3,
out_classes=1,
num_encoding_blocks=3,
out_channels_first_layer=16,
normalization='batch',
padding=True,
pooling_type='max', # max avg AdaptiveMax AdaptiveAvg
upsampling_type='trilinear',
residual=False,
dropout=False,
monte_carlo_dropout=0.5)
elif network_name == 'unet':
self.model = SmallUnet(in_channels=1, out_channels=1)
elif network_name == 'ConvN':
conv_block = par_model['conv_block']
dropout, drop_conv, batch_norm = par_model['dropout'], par_model[
'drop_conv'], par_model['batch_norm']
linear_block = par_model['linear_block']
output_fnc = par_model[
'output_fnc'] if 'output_fnc' in par_model else None
self.model = ConvN_FC3(in_size=in_size,
conv_block=conv_block,
linear_block=linear_block,
dropout=dropout,
drop_conv=drop_conv,
batch_norm=batch_norm,
output_fnc=output_fnc)
network_name += '_C{}_{}_Lin{}_{}_D{}_DC{}'.format(
np.abs(conv_block[0]), conv_block[-1], linear_block[0],
linear_block[-1], dropout, drop_conv)
if output_fnc is not None:
network_name += '_fnc_{}'.format(output_fnc)
if batch_norm:
network_name += '_BN'
if self.validation_droupout:
network_name += '_VD'
self.res_name += '_Size{}_{}_Loss_{}_lr{}'.format(
in_size[0], network_name, losstype, lr)
if 'Adam' not in optim_name: #only write if not default Adam
self.res_name += '_{}'.format(optim_name)
self.res_dir += self.res_name + '/'
if res_model_file is not None: #to avoid handeling batch size and num worker used for model training
self.res_dir, self.res_name = get_parent_path(res_model_file)
if not os.path.isdir(self.res_dir): os.mkdir(self.res_dir)
self.log = get_log_file(self.res_dir + '/' + log_filename)
self.log.info(self.log_string)
if losstype == 'MSE':
self.loss = tnn.MSELoss()
elif losstype == 'L1':
self.loss = tnn.L1Loss()
elif losstype == 'ssim':
self.loss = SSIM3D()
elif losstype == 'ssim_dist':
self.loss = SSIM3D(distance=2)
elif losstype == 'BCE':
self.loss = tnn.BCELoss()
elif losstype == 'BCElogit':
self.loss = tnn.BCEWithLogitsLoss()
if self.cuda:
self.model = self.model.cuda()
self.loss = self.loss.cuda()
device = "cuda"
else:
device = 'cpu'
if verbose:
self.log.info(
summary(self.model, (1, in_size[0], in_size[1], in_size[2]),
device=device,
batch_size=1))
self.ep_start, self.last_model_saved = load_existing_weights_if_exist(
self.res_dir,
self.model,
log=self.log,
device=device,
res_model_file=res_model_file)
if "Adam" in optim_name:
self.optimizer = optim.Adam(self.model.parameters(), lr=lr)
elif "SGD" in optim_name:
self.optimizer = optim.SGD(self.model.parameters(),
lr=lr,
momentum=0.5)
import os
resdir_mvt = res_dir + '/mvt_param/'
resdir_fig = res_dir + '/fig/'
try: #on cluster, all job are doing the mkdir at the same time ...
if not os.path.isdir(resdir_mvt): os.mkdir(resdir_mvt)
if not os.path.isdir(resdir_fig): os.mkdir(resdir_fig)
except:
pass
transfo = get_motion_transform(type=motion_type)
torch.manual_seed(seed)
np.random.seed(seed)
dir_img = get_parent_path([fin])[0]
fm = gfile(dir_img, '^mask', {"items": 1})
fp1 = gfile(dir_img, '^p1', {"items": 1})
fp2 = gfile(dir_img, '^p2', {"items": 1})
if len(fm) == 0: #may be in cat12 subdir (like for HCP)
fm = gfile(dir_img, '^brain_T1', {"items": 1})
#dir_cat = gdir(dir_img,'cat12')
#fm = gfile(dir_cat, '^mask_brain', {"items": 1})
#fp1 = gfile(dir_cat, '^p1', {"items": 1})
#fp2 = gfile(dir_cat, '^p2', {"items": 1})
one_suj = {'image': Image(fin, INTENSITY), 'brain': Image(fm[0], LABEL)}
if len(fp1) == 1:
one_suj['p1'] = Image(fp1[0], LABEL)
if len(fp2) == 1:
one_suj['p2'] = Image(fp2[0], LABEL)
def get_dataset_from_option(options):
fin = options.image_in
dir_sample = options.sample_dir
add_affine_zoom, add_affine_rot = options.add_affine_zoom, options.add_affine_rot
batch_size, num_workers = options.batch_size, options.num_workers
doit = do_training('/tmp/', 'not_use', verbose=True)
# adding transformation
tc = []
name_suffix = ''
#Attention pas de _ dans le name_suffix
if options.add_cut_mask > 0:
target_shape, mask_key = (182, 218, 182), 'brain'
tc = [
CropOrPad(target_shape=target_shape, mask_name=mask_key),
]
name_suffix += '_tCropBrain'
if add_affine_rot > 0 or add_affine_zoom > 0:
if add_affine_zoom == 0: add_affine_zoom = 1 #0 -> no affine so 1
tc.append(
RandomAffine(scales=(add_affine_zoom, add_affine_zoom),
degrees=(add_affine_rot, add_affine_rot),
image_interpolation=Interpolation.NEAREST))
name_suffix += '_tAffineS{}R{}'.format(add_affine_zoom, add_affine_rot)
# for hcp should be before RescaleIntensity
mask_brain = False
if options.add_mask_brain:
tc.append(ApplyMask(masking_method='brain'))
name_suffix += '_tMaskBrain'
mask_brain = True
if options.add_rescal_Imax:
tc.append(RescaleIntensity(percentiles=(0, 99)))
name_suffix += '_tRescale-0-99'
if options.add_elastic1:
tc.append(get_motion_transform(type='elastic1'))
name_suffix += '_tElastic1'
if options.add_bias:
tc.append(RandomBiasField())
name_suffix += '_tBias'
if len(name_suffix) == 0:
name_suffix = '_Raw'
target = None
if len(tc) == 0: tc = None
add_to_load, add_to_load_regexp = None, None
if len(dir_sample) > 0:
print('loading from {}'.format(dir_sample))
if options.add_orig:
add_to_load, add_to_load_regexp = 'original', 'notused'
data_name = get_parent_path(dir_sample)[1]
if mask_brain and 'hcp' in data_name:
add_to_load_regexp = 'brain_T'
if add_to_load is None:
add_to_load = 'brain'
else:
add_to_load += 'brain'
doit.set_data_loader(batch_size=batch_size,
num_workers=num_workers,
load_from_dir=dir_sample,
transforms=tc,
add_to_load=add_to_load,
add_to_load_regexp=add_to_load_regexp)
name_suffix = 'On_' + data_name + name_suffix
target = options.target #'ssim' #suppose that if from sample, it should be simulation so set target
else:
print('working on ')
for ff in fin:
print(ff)
doit.set_data_loader_from_file_list(fin,
transforms=tc,
batch_size=batch_size,
num_workers=num_workers,
mask_key=mask_key,
mask_regex='^mask')
return doit, name_suffix, target
plt.ioff()
resdir_fig = res_dir + '/fig/'
resdir_mvt = res_dir + '/mvt_param/'
fres = res_dir + '/res_motion.csv'
if not os.path.exists(fres):
print('found no result file {} \n So building it'.format(fres))
print(
'loading {} sample from {} \n estimated time {} h (for 2 iter/s)'.
format(len(td), load_from_dir,
len(td) / (2 * 60 * 60)))
start = time.time()
res, extra_info = pd.DataFrame(), dict()
for ii, sample in enumerate(tqdm(td)):
ff = sample['mvt_csv']
fcsv_name = get_parent_path(ff)[1][0]
extra_info = {'mvt_csv': ff, 'sample': fsaved[ii]}
res = doit.add_motion_info(sample, res, extra_info)
res.to_csv(fres)
print('saving {}'.format(fres))
print('done csv in {}'.format((time.time() - start) / 60 / 60))
if do_plot_fig:
start = time.time()
res = pd.read_csv(fres)
param = res.ssim.values
nb_val = 30 #96
pind = np.argsort(param)
p0 = np.percentile(pind, 10) #first 10
pm1, pm2 = np.percentile(pind, 40), np.percentile(pind, 60)
rlabel = rlabel.sort_index() # alphabetic order
labelsujid = rlabel.index
rr = rlabell.reindex(rlabel.index).loc[:, ['lesion_PV', 'lesion_WM']]
rlabel = pd.concat([rlabel, rr], axis=1, sort=True)
# reorder the label as res[0]
# rlabel = rlabel.loc[sujid]
# ytrue = rlabel.QCOK.values
ytrue = rlabel.globalQualitative.values
print_accuracy_df(rlabel, ytrue)
# prediction mriqc
rd = '/network/lustre/dtlake01/opendata/data/ABIDE/mriqc_data/retrain'
resfile = gfile(rd, 'data_CATI.*csv$')
resname = get_parent_path(resfile, 1)[1]
res = [pd.read_csv(f) for f in resfile]
for ii, rr in enumerate(res):
sujid = []
for ff in rr.subject_id:
dd = ff.split('/')
if dd[-1] is '': dd.pop()
nn = len(dd)
sujid.append(dd[nn - 3] + '+' + dd[nn - 2] + '+' + dd[nn - 1])
rr.index = sujid
res[ii] = rr.loc[labelsujid] # rr.loc[sujid[::-1]]
print_accuracy(res,
resname,
ytrue,
from torchvision.transforms import Compose
import torchio as tio
from torchio.data.io import write_image, read_image
from utils_file import get_parent_path, gfile, gdir
from slices_2 import do_figures_from_file
from utils import reduce_name_list, remove_string_from_name_list
from utils_plot_results import get_ep_iter_from_res_name, plot_resdf, plot_train_val_results, \
transform_history_to_factor, parse_history
import commentjson as json
#res_valOn
dd = gfile('/network/lustre/dtlake01/opendata/data/ds000030/rrr/CNN_cache_new',
'_')
dir_fig = '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/NN_regres_motion/figure/motion_regress/eval2/'
dir_fig = '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/NN_regres_random_noise/figure2/'
data_name_list = get_parent_path(dd)[1]
dres_reg_exp, figname = ['Reg.*D0_DC0'], ['noise']
dres_reg_exp, figname = ['.*hcp.*ms', '.*hcp.*T1', 'cati.*ms', 'cati.*T1'
], ['hcp_ms', 'hcp_T1', 'cati_ms', 'cati_T1']
sns.set(style="whitegrid")
csv_regex = 'res_valOn_'
for rr, fign in zip(dres_reg_exp, figname):
dres = gdir(dqc, rr)
resname = get_parent_path(dres)[1]
resname = remove_string_from_name_list(resname, [
'RegMotNew_', 'Size182_ConvN_C16_256_Lin40_50_', '_B4',
'_Loss_L1_lr0.0001', '_nw0_D0'
])
resname = [fign + '_' + zz for zz in resname]
# parameters
root_dir = '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/NN_regres_motion/'
prefix = "/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/job/job_eval/"
name = 'cati_all_ms_ela1_train200'
name = 'cati_all_ms_train_cati_ms'
name = 'cati_all_T1_train_cati_T1'
model = root_dir + 'RegMotNew_ela1_train200_hcp400_ms_B4_nw0_Size182_ConvN_C16_256_Lin40_50_D0_BN_Loss_L1_lr0.0001/model_ep3_it9500.pt'
model = root_dir + 'RegMotNew_ela1_train_cati_ms_B4_nw0_Size182_ConvN_C16_256_Lin40_50_D0_BN_Loss_L1_lr0.0001/model_ep8_it1249.pt'
model = root_dir + 'RegMotNew_ela1_train_cati_T1_B4_nw0_Size182_ConvN_C16_256_Lin40_50_D0_BN_Loss_L1_lr0.0001/model_ep8_it1249.pt'
split = 10
model_name = get_parent_path(model)[1][:-3]
if 'cati_all_ms' in name:
res = pd.read_csv(
'/home/romain.valabregue/datal/QCcnn/CATI_datasets/cati_cenir_all_ms.csv'
)
elif 'cati_all_T1' in name:
res = pd.read_csv(
'/home/romain.valabregue/datal/QCcnn/CATI_datasets/cati_cenir_all_T1.csv'
)
fin_all = res.filename
nb_jobs = len(fin_all) // split + 1
jobs = []
for njob in range(0, nb_jobs):
fin = fin_all[njob * split:(njob + 1) * split]
from torchio.transforms import RandomMotionFromTimeCourse, RandomAffine, \
CenterCropOrPad, RandomElasticDeformation, RandomElasticDeformation, CropOrPad, RandomNoise
from torchio import Image, ImagesDataset, transforms, INTENSITY, LABEL, Interpolation, Subject
from utils_file import get_parent_path, gfile, gdir
from doit_train import do_training, get_motion_transform
from slices_2 import do_figures_from_file
from utils import reduce_name_list, get_ep_iter_from_res_name
#Explore csv results
dqc = [
'/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/NN_regres_motion'
]
dres = gdir(dqc, 'RegMotNew.*train_hcp400_ms.*0001')
dres = gdir(dqc, 'RegMotNew.*hcp400_ms.*B4.*L1.*0001')
dres = gdir(dqc, 'R.*')
resname = get_parent_path(dres)[1]
#sresname = [rr[rr.find('hcp400_')+7: rr.find('hcp400_')+17] for rr in resname ]; sresname[2] += 'le-4'
sresname = resname
commonstr, sresname = reduce_name_list(sresname)
print('common str {}'.format(commonstr))
target = 'ssim'
target_scale = 1
#target='random_noise'; target_scale=10
legend_str = []
col = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'w']
for ii, oneres in enumerate(dres):
fresV = gfile(oneres, 'res_val')
fresT = gfile(oneres, 'train.*csv')
is_train = False if len(fresT) == 0 else True
dir_cache = '/network/lustre/dtlake01/opendata/data/ds000030/rrr/CNN_cache_new/'
jobs = []
scriptsDir = '/network/lustre/iss01/cenir/software/irm/toolbox_python/romain/torchQC'
for data_name_val in name_list_val:
dir_sample = '{}/{}/'.format(dir_cache, data_name_val)
for saved_model in saved_models:
for opt_test in options_test:
py_options = '--use_gpu 0 --saved_model {} {} '.format(
saved_model, opt_test)
model_name = get_parent_path(saved_model)[1]
cmd_init = '\n'.join([
"#source /network/lustre/iss01/cenir/software/irm/bin/python_path3.6",
"#source activate pytorch1.2",
"python " + scriptsDir + "/do_eval_model.py \\",
py_options + " \\"
])
job = '\n'.join([cmd_init, ' --sample_dir ' + dir_sample])
jobs.append(job)
params = dict()
params['output_directory'] = prefix + '/jobs/' + job_id
params['scripts_to_copy'] = scriptsDir #+ '/*.py'
dir_sample = options.sample_dir
target_shape, mask_key = (182, 218, 182), 'brain'
tc = [
CropOrPad(target_shape=target_shape, mask_name=mask_key),
]
doit = do_training('/tmp/', 'not_use', verbose=True)
if len(dir_sample) > 0:
print('loading from {}'.format(dir_sample))
doit.set_data_loader(batch_size=batch_size,
num_workers=num_workers,
load_from_dir=dir_sample,
transforms=tc)
name += '_' + get_parent_path(dir_sample)[1]
else:
print('working on ')
for ff in fin:
print(ff)
doit.set_data_loader_from_file_list(fin,
transforms=tc,
batch_size=batch_size,
num_workers=num_workers,
mask_key=mask_key,
mask_regex='^mask')
out_name = 'eval_num_{:04d}'.format(int(val_number))
subdir = 'eval_{}_{}'.format(name, get_parent_path(saved_model)[1][:-3])
