def load_existing_weights_if_exist(resdir, model, model_name='model', log=None, device='cuda', index_mod=-1, res_model_file=None): from utils_file import gfile, get_parent_path ep_start = 0 if res_model_file is None: resume_mod = gfile(resdir, '.*pt$') else: resume_mod = [res_model_file] if len(resume_mod) > 0: dir_mod, fn = get_parent_path(resume_mod) ffn = [ff[ff.find('_ep') + 3:-3] for ff in fn] key_list = [] for fff, fffn in zip(ffn, fn): if '_it' in fff: ind = fff.find('_it') ep = int(fff[0:ind]) it = int(fff[ind + 3:]) else: ep = int(fff) it = 100000000 key_list.append([fffn, ep, it]) aa = np.array(sorted(key_list, key=lambda x: (x[1], x[2]))) name_sorted, ep_sorted = aa[:, 0], aa[:, 1] ep_start = int(ep_sorted[index_mod]) thelast = dir_mod[0] + '/' + name_sorted[index_mod] log.info('RESUME model from epoch {} weight loaded from {}'.format( ep_start, thelast)) tl = torch.load(thelast, map_location=device) if model_name not in tl: model_name = list(tl.items())[0][0] prefix = 'model.' state_dict = tl[model_name] aa = next(iter(state_dict)) if prefix in aa: new_state_dict = OrderedDict() for k, v in state_dict.items(): name = k[len(prefix):] # remove 'module.' of dataparallel new_state_dict[name] = v model.load_state_dict(new_state_dict) else: model.load_state_dict(tl[model_name]) else: log.info('New training starting epoch {}'.format(ep_start)) thelast = resdir log.info('Resdir is {}'.format(resdir)) return ep_start, get_parent_path([thelast])[1][0]
def set_model_from_file(self, file_name, cuda): resdir_name = get_parent_path(file_name, 2)[1] print('loading {} \nfrom dir {}'.format( get_parent_path(file_name)[1], resdir_name)) if 'ConvN_C16_256_Lin40_50' in resdir_name: conv_block, linear_block = [16, 32, 64, 128, 256], [40, 50] network_name = 'ConvN' else: raise ('did not recognise model type') if 'L1' in resdir_name: losstype = 'L1' elif 'MSE' in resdir_name: losstype = 'MSE' if '_Size182' in resdir_name: in_size = [182, 218, 182] batch_norm = True if '_BN_' in resdir_name else False ind_drop = resdir_name.find('_D') substr = resdir_name[ind_drop + 2:] ii = substr.find('_') dropout = float(substr[:ii]) drop_conv = 0 if '_DC' in resdir_name: ind_drop = resdir_name.find('_DC') substr = resdir_name[ind_drop + 3:] ii = substr.find('_') drop_conv = float(substr[:ii]) ind_lr = resdir_name.find('_lr') lr = float(resdir_name[ind_lr + 3:]) par_model = { 'network_name': network_name, 'losstype': losstype, 'lr': lr, 'conv_block': conv_block, 'linear_block': linear_block, 'dropout': dropout, 'drop_conv': drop_conv, 'batch_norm': batch_norm, 'in_size': in_size, 'cuda': cuda, 'max_epochs': 1 } self.set_model(par_model, res_model_file=file_name, verbose=False, log_filename='eval.log')
def __getitem__(self, index: int) -> Subject: if not isinstance(index, int): raise ValueError(f'Index "{index}" must be int, not {type(index)}') if self.load_from_dir: subject = torch.load(self._subjects[index]) if self.add_to_load is not None: #print('adding subject with {}'.format(self.add_to_load)) ii = subject.get_images() image_path = ii[0]['path'] if 'original' in self.add_to_load: #print('adding original subject') ss = Subject(image = Image(image_path, INTENSITY)) # sss = self._get_sample_dict_from_subject(ss) #sss = copy.deepcopy(ss) sss = ss subject['original'] = sss['image'] if self.add_to_load=='original': #trick to use both orig and mask :hmmm.... add_to_load = None else: add_to_load = self.add_to_load[8:] else: add_to_load = self.add_to_load if add_to_load is not None: image_add = gfile(get_parent_path(image_path), self.add_to_load_regexp)[0] #print('adding image {} to {}'.format(image_add,self.add_to_load)) ss = Subject(image = Image(image_add, LABEL)) #sss = self._get_sample_dict_from_subject(ss) #sss = copy.deepcopy(ss) sss = ss hh = subject.history for hhh in hh: if 'RandomElasticDeformation' in hhh[0]: from torchio.transforms import RandomElasticDeformation num_cp = hhh[1]['coarse_grid'].shape[1] rr = RandomElasticDeformation(num_control_points=num_cp) sss = rr.apply_given_transform(sss, hhh[1]['coarse_grid']) subject[add_to_load] = sss['image'] #print('subject with keys {}'.format(subject.keys())) else: subject = self._subjects[index] subject = copy.deepcopy(subject) # cheap since images not loaded yet if self.load_getitem: subject.load() # Apply transform (this is usually the bottleneck) if self._transform is not None: subject = self._transform(subject) if self.save_to_dir is not None: res_dir = self.save_to_dir fname = res_dir + '/subject{:05d}'.format(index) if 'image_orig' in subject: subject.pop('image_orig') torch.save(subject, fname + '_subject.pt') return subject
def write_cati_csv(): import pandas as pd data_path = '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/CATI_datasets/' fcsv = data_path + 'all_cati.csv' res = pd.read_csv(fcsv) ser_dir = res.cenir_QC_path ser_dir = res.cenir_QC_path[res.globalQualitative > 3].values dcat = gdir(ser_dir, 'cat12') fT1 = gfile(dcat, '^s.*nii') fms = gfile(dcat, '^ms.*nii') fs_brain = gfile(dcat, '^brain_s.*nii') # return fT1, fms, fs_brain ind_perm = np.random.permutation(range(0, len(fT1))) itrain = ind_perm[0:100] ival = ind_perm[100:] dd = pd.DataFrame({'filename': fT1}) dd.to_csv(data_path + 'cati_cenir_QC4_all_T1.csv', index=False) dd.loc[ival, :].to_csv(data_path + 'cati_cenir_QC4_val_T1.csv', index=False) dd.loc[itrain, :].to_csv(data_path + 'cati_cenir_QC4_train_T1.csv', index=False) dd = pd.DataFrame({'filename': fms}) dd.to_csv(data_path + 'cati_cenir_QC4_all_ms.csv', index=False) dd.loc[ival, :].to_csv(data_path + 'cati_cenir_QC4_val_ms.csv', index=False) dd.loc[itrain, :].to_csv(data_path + 'cati_cenir_QC4_train_ms.csv', index=False) dd = pd.DataFrame({'filename': fs_brain}) dd.to_csv(data_path + 'cati_cenir_QC4_all_brain.csv', index=False) dd.loc[ival, :].to_csv(data_path + 'cati_cenir_QC4_val_brain.csv', index=False) dd.loc[itrain, :].to_csv(data_path + 'cati_cenir_QC4_train_brain.csv', index=False) dd = pd.DataFrame({'filename': fT1}) dd.to_csv(data_path + 'cati_cenir_all_T1.csv', index=False) dd = pd.DataFrame({'filename': fms}) dd.to_csv(data_path + 'cati_cenir_all_ms.csv', index=False) dd = pd.DataFrame({'filename': fs_brain}) dd.to_csv(data_path + 'cati_cenir_all_brain.csv', index=False) #add brain mask in csv allcsv = gfile('/home/romain.valabregue/datal/QCcnn/CATI_datasets', 'cati_cenir.*csv') for onecsv in allcsv: res = pd.read_csv(onecsv) resout = onecsv[:-4] + '_mask.csv' fmask = [] for ft1 in res.filename: d = get_parent_path(ft1)[0] fmask += gfile(d, '^mask', opts={"items": 1}) res['brain_mask'] = fmask res.to_csv(resout, index=False)
def get_train_and_val_csv(names='', root_fs='lustre'): return_list = True if isinstance(names, str): names = [names] return_list = False print('name is {}'.format(type(names))) if root_fs == 'lustre': data_path_hcp = '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/' elif root_fs == 'le70': data_path_hcp = '/data/romain/HCPdata/' data_path_cati = '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/CATI_datasets/' fcsv_train, fcsv_val = [], [] for name in names: if 'hcp' in name: if 'T1' in name: fname_train, fname_val = 'Motion_T1_train_hcp400.csv', 'Motion_T1_val_hcp200.csv' elif 'brain_ms' in name: fname_train, fname_val = 'healthy_brain_ms_train_hcp400.csv', 'healthy_brain_ms_val_hcp200.csv' elif 'ms' in name: fname_train, fname_val = 'healthy_ms_train_hcp400.csv', 'healthy_ms_val_hcp200.csv' else: print('can not guess which DATA from {}'.format(name)) raise file_train = data_path_hcp + fname_train file_val = data_path_hcp + fname_val elif 'cati' in name: if 'T1' in name: fname_train, fname_val = 'cati_cenir_QC4_train_T1.csv', 'cati_cenir_QC4_val_T1.csv' elif 'brain' in name: fname_train, fname_val = 'cati_cenir_QC4_train_brain.csv', 'cati_cenir_QC4_val_brain.csv' elif 'i_ms' in name: fname_train, fname_val = 'cati_cenir_QC4_train_ms.csv', 'cati_cenir_QC4_val_ms.csv' else: print('can not guess which DATA from {}'.format(name)) raise file_train = data_path_cati + fname_train file_val = data_path_cati + fname_val else: print('can not guess which DATA from {}'.format(name)) raise print('data {}\nfor {} \t found {} {} '.format( get_parent_path([file_train])[0][0], name, fname_train, fname_val)) fcsv_train.append(file_train) fcsv_val.append(file_val) if return_list: return fcsv_train, fcsv_val else: return fcsv_train[0], fcsv_val[0]
def get_subject_list_from_file_list(fin, mask_regex=None, mask_key='brain'): subjects_list = [] for ff in fin: one_suj = {'image': Image(ff, INTENSITY)} if mask_regex is not None: dir_file = get_parent_path(ff)[0] fmask = gfile(dir_file, mask_regex, {"items": 1}) one_suj[mask_key] = Image(fmask[0], LABEL) subjects_list.append(Subject(one_suj)) return subjects_list
parser.add_option("-w", "--saved_model", action="store", dest="saved_model", default='', help="full path of the model's weights file ") parser.add_option("--use_gpu", action="store", dest="use_gpu", default=0, type="int", help="0 means no gpu 1 to 4 means gpu device (0) ") (options, args) = parser.parse_args() log = get_log_file() name, val_number = options.out_name, options.val_number saved_model = options.saved_model cuda = True if options.use_gpu > 0 else False if val_number<0: out_name = name subdir = None #'eval_rrr__{}_{}'.format(name) else: out_name = 'eval_num_{:04d}'.format(val_number) subdir = 'eval_{}_{}'.format(name, get_parent_path(saved_model)[1][:-3]) doit, name_suffix, target = get_dataset_from_option(options) out_name += name_suffix doit.set_model_from_file(saved_model, cuda=cuda) doit.validation_droupout = True doit.eval_regress_motion(999, 99, basename=out_name, subdir=subdir, target=target)
def get_sujname_from_label_filename(ligne, col_name): cell_content = ligne[col_name] array_path = eval(cell_content) return get_parent_path(str(array_path[0]),level=4)[1]
res = gdir('/home/romain.valabregue/datal/PVsynth/training/RES_14mm_tissue','data') res = gdir('/home/romain.valabregue/datal/PVsynth/jzay/training/RES1mm_prob','pve_synth_mod3_P128$') res = gdir('/home/romain.valabregue/datal/PVsynth/jzay/training/RES1mm_prob','aniso') res = gdir(res,'results_cluster') res = ['/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/PVsynth/jzay/training/RES1mm_prob/pve_synth_mod3_P128_aniso_LogLkd_reg_multi/result', '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/PVsynth/jzay/training/RES1mm_prob/pve_synth_mod3_P128_aniso_LogLkd_reg_unis_lam1/results_cluster', '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/PVsynth/jzay/training/RES1mm_prob/pve_synth_mod3_P128_aniso_LogLkd_classif/results_cluster', ] res = ['/home/romain.valabregue/datal/PVsynth/jzay/training/RES1mm_prob/pve_synth_mod3_P128/results_cluster/'] report_learning_curves(res) #explore synthetic data histogram results_dirs = glob.glob('/home/romain.valabregue/datal/PVsynth/RES_1.4mm/t*/513130') f=gfile(results_dirs,'^5.*nii') resname = get_parent_path(results_dirs,2)[1] resfig = '/home/romain.valabregue/datal/PVsynth/figure/volume_synth/' for i, ff in enumerate(f): img = nb.load(ff) data = img.get_fdata(dtype=np.float32) fig = plt.figure(resname[i]) hh = plt.hist(data.flatten(), bins=500) axes = plt.gca() axes.set_ylim([0 , 40000]) #fig.savefig(resfig + resname[i] + '.png') #concat eval.csv res = '/home/romain.valabregue/datal/PVsynth/eval_cnn/RES_14mm_tissue/dataS*/*/eval.csv'
def report_learning_curves(results_dirs, save=False): """ Plot error curves from record files and save plot to jpeg file. """ def plot_losses(pattern, color, label): files = glob.glob(results_dir + pattern) files.sort(key=os.path.getmtime) losses, ymean, qt05, qt95 = [], [], [], [] nb_iter_list = [] nb_iter_mem = -1 for file in files: df = pd.read_csv(file, index_col=0) loss = df['loss'].values ymean.append(np.mean(loss)) qt05.append(np.quantile(loss, 0.05)) qt95.append(np.quantile(loss, 0.95)) #losses.append(loss) nb_iter = df.shape[0] if nb_iter_mem >= 0: if not(nb_iter_mem == nb_iter): print('\t {} \t {} iter previous {} '.format(os.path.basename(file), nb_iter, nb_iter_mem,)) nb_iter_mem = nb_iter nb_iter_list.append(nb_iter) #plt.plot(np.mean(losses, axis=1), color=color, label=label) #plt.plot(np.quantile(losses, 0.95, axis=1), '--', color=color) #plt.plot(np.quantile(losses, 0.05, axis=1), '--', color=color) plt.plot(ymean, color=color, label=label) plt.plot(qt95, '--', color=color) plt.plot(qt05, '--', color=color) return [np.max(nb_iter_list), np.min(nb_iter_list), ymean] if isinstance(results_dirs,str): results_dirs = [results_dirs] train_loss, resname_list = [], [] for results_dir in results_dirs: resname = get_parent_path(results_dir, level=2)[1] print(resname) resname_list.append(resname) plt.figure(resname) iter_max, iter_min, ymeans = plot_losses('/Train_ep*.csv', 'blue', 'train mean loss') train_loss.append(ymeans) #plot_losses('/Val_ep*.csv', 'green', 'val mean loss') if iter_max==iter_min: plt.xlabel('epch ( {} iter)'.format(iter_max)) else: plt.xlabel('epoch ( {} iter {})'.format(iter_max, iter_min)) plt.legend() plt.title('Training and validation error curves') plt.show() if save: plt.savefig(results_dir + '/{}loss_curves.jpeg'.format(resname)) plt.figure('all') for tt in train_loss: plt.plot(tt) plt.legend(resname_list)
def plot_train_val_results(dres, train_csv_regex='Train.*csv', val_csv_regex='Val.*csv', prediction_column_name='prediction', target_column_name='targets', target_scale=1, fign='Res', sresname=None): legend_str=[] col = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'w'] for ii, oneres in enumerate(dres): fresT = gfile(oneres, train_csv_regex) fresV=gfile(oneres, val_csv_regex) if len(fresV)==0: print('{} empty dir {} '.format(colored('Skiping','red'), get_parent_path(oneres)[1])) continue is_train = False if len(fresT)==0 else True if is_train: resT = [pd.read_csv(ff) for ff in fresT] resdir, resname = get_parent_path(fresV) nbite = len(resT[0]) if is_train else 80000 fresV_sorted, b, c = get_ep_iter_from_res_name(resname, nbite) ite_tot = c+b*nbite ite_tottt = np.hstack([0, ite_tot]) print(ite_tot) resV = [pd.read_csv(resdir[0] + '/' + ff) for ff in fresV_sorted] df_val = pd.concat(resV, ignore_index=True, sort=False) for rr in resV: if 'sample_time' not in rr: rr['sample_time'] = rr['batch_time'] / 4 #for old result always runed with batchsize 4 if isinstance(rr[prediction_column_name][0], str): rr[prediction_column_name] = rr[prediction_column_name].apply( lambda s: convert_string_array_to_array(s)) rr[target_column_name] = rr[target_column_name].apply( lambda s: convert_string_array_to_array(s)) if is_train: for rr in resT: if 'sample_time' not in rr: rr['sample_time'] = rr['batch_time'] / 4 if isinstance(rr[prediction_column_name][0], str): rr[prediction_column_name] = rr[prediction_column_name].apply( lambda s: convert_string_array_to_array(s)) rr[target_column_name] = rr[target_column_name].apply( lambda s: convert_string_array_to_array(s)) if is_train: df_train = pd.concat(resT, ignore_index=True, sort=False) errorT = np.abs(df_train.loc[:,prediction_column_name].values -df_train.loc[:, target_column_name].values*target_scale) train_time = df_train.loc[:,'sample_time'] #average between validation point itte_tot LmTrain = [np.mean(errorT[ite_tottt[ii]:ite_tottt[ii+1]]) for ii in range(0, len(ite_tot)) ] TimeTrain = [np.mean(train_time[ite_tottt[ii]:ite_tottt[ii + 1]]) for ii in range(0, len(ite_tot))] LmVal = [np.mean(np.abs(rr.loc[:,prediction_column_name]-rr.loc[:, target_column_name].values*target_scale)) for rr in resV] #LmVal = np.mean(np.abs(df_val.loc[:,prediction_column_name].values - df_val.loc[:,target_column_name].values*target_scale)) TimeVal = [np.mean(rr.loc[:,'sample_time']) for rr in resV] plt.figure('MeanL1_'+fign); legend_str.append('V{}'.format(sresname[ii])); if is_train: legend_str.append('T{}'.format(sresname[ii])) plt.plot(ite_tot, LmVal,'--',color=col[ii]) if is_train: plt.plot(ite_tot, LmTrain,color=col[ii], linewidth=6) plt.figure('Time_'+fign); plt.plot(ite_tot, TimeVal,'--',color=col[ii]) if is_train: plt.plot(ite_tot, TimeTrain,color=col[ii], linewidth=6) #print some summary information on the results if not is_train: TimeTrain=0 nb_res = len(resT) if is_train else 0 np_iter = len(resT[0]) if is_train else 0 totiter, mbtt, mbtv = ite_tot[-1] / 1000, np.nanmean(TimeTrain), np.mean(TimeVal) tot_time = nb_res * np_iter * mbtt + len(resV) * len(resV[0]) * mbtv percent_train = nb_res * np_iter * mbtt / tot_time tot_time_day = np.floor( tot_time/60/60/24 ) tot_time_hour = (tot_time - tot_time_day*24*60*60) / 60/60 print('Result : {} \t {} '.format( colored(get_parent_path(resdir[0])[1], 'green'), sresname[ii] )) print('\t{} epoch of {} vol {} val on {} vol Tot ({:.1f}%train) {} d {:.1f} h'.format( nb_res, np_iter, len(resV),len(resV[0]), percent_train, tot_time_day, tot_time_hour )) fj = gfile(oneres,'data.json') if len(fj)==1: data_struc = cc.read_json(fj[0]) bs, nw = data_struc['batch_size'], data_struc['num_workers'] else: bs, nw = 0, -1 print('\tBatch size {} \tNum worker {} \t{:.1f} mille iter \t train/val meanTime {:.2f} / {:.2f} '.format\ (bs, nw, totiter, mbtt, mbtv)) plt.figure('MeanL1_'+fign); plt.legend(legend_str); plt.grid() ff=plt.gcf();ff.set_size_inches([15, 7]); #ff.tight_layout() plt.subplots_adjust(left=0.05, right=1, bottom=0.05, top=1, wspace=0, hspace=0) plt.ylabel('L1 loss') plt.figure('Time_'+fign); plt.legend(legend_str); plt.grid() plt.ylabel('time in second')
def get_pandadf_from_res_valOn_csv(dres, resname, csv_regex='res_valOn', data_name_list=None, select_last=None, target='ssim', target_scale=1): if len(dres) != len(resname) : raise('length problem between dres and resname') resdf_list = [] for oneres, resn in zip(dres, resname): fres_valOn = gfile(oneres, csv_regex) print('Found {} <{}> for {} '.format(len(fres_valOn), csv_regex, resn)) if len(fres_valOn) == 0: continue ftrain = gfile(oneres, 'res_train_ep01.csv') rrt = pd.read_csv(ftrain[0]) nb_it = rrt.shape[0]; resdir, resname_val = get_parent_path(fres_valOn) resname_sorted, b, c = get_ep_iter_from_res_name(resname_val, 0) if select_last is not None: if select_last<0: resname_sorted = resname_sorted[select_last:] else: nb_iter = b*nb_it+c resname_sorted = resname_sorted[np.argwhere(nb_iter > select_last)[1:8]] resV = [pd.read_csv(resdir[0] + '/' + ff) for ff in resname_sorted] resdf = pd.DataFrame() for ii, fres in enumerate(resname_sorted): iind = [i for i, s in enumerate(data_name_list) if s in fres] if len(iind) ==1: #!= 1: raise ("bad size do not find which sample") data_name = data_name_list[iind[0]] else: data_name = 'res_valds' iind = fres.find(data_name) ddn = remove_extension(fres[iind + len(data_name) + 1:]) new_col_name = 'Mout_' + ddn iind = ddn.find('model_ep') if iind==0: transfo='raw' else: transfo = ddn[:iind - 1] if transfo[0] == '_': #if start with _ no legend ... ! transfo = transfo[1:] model_name = ddn[iind:] aa, bb, cc = get_ep_iter_from_res_name([fres], nb_it) nb_iter = bb[0] * nb_it + cc[0] rr = resV[ii].copy() rr['evalOn'], rr['transfo'] = data_name, transfo rr['model_name'], rr['submodel_name'], rr['nb_iter'] = resn, model_name, str(nb_iter) rr[target] = rr[target] * target_scale resdf = pd.concat([resdf, rr], axis=0, sort=True) resdf['error'] = resdf[target] - resdf['model_out'] resdf['error_abs'] = np.abs(resdf[target] - resdf['model_out']) resdf_list.append(resdf) return resdf_list
from scipy.linalg import logm, expm weights, matrices = ss[0], affs logs = [w * logm(A) for (w, A) in zip(weights, matrices)] logs = np.array(logs) logs_sum = logs.sum(axis=0) expm(logs_sum/np.sum(weights, axis=0) ) #a 10-2 pres c'est bien l'identite ! rp_files = gfile('/data/romain/HCPdata/suj_274542/Motion_ms','^rp') rp_files = gfile('/data/romain/HCPdata/suj_274542/mot_separate','^rp') rpf = rp_files[10] res = pd.DataFrame() for rpf in rp_files: dirpath,name = get_parent_path([rpf]) fout = dirpath[0] + '/check/'+name[0][3:-4] + '.nii' t = RandomMotionFromTimeCourse(fitpars=rpf, nufft=True, oversampling_pct=0, keep_original=True, verbose=True) dataset = ImagesDataset(suj, transform=t) sample = dataset[0] dicm = sample['T1']['metrics'] dicm['fname'] = fout res = res.append(dicm, ignore_index=True) dataset.save_sample(sample, dict(T1=fout)) fit_pars = sample['T1']['fit_pars'] plt.figure; plt.plot(fit_pars[3:].T) plt.figure; plt.plot(fit_pars.T)
filename = res+k+'.json' with open(filename, 'w') as file: json.dump(conf_all[k], file, indent=4, sort_keys=True) filename = res+'grid_search.json' with open(filename, 'w') as file: json.dump(gs, file, indent=4, sort_keys=True) # generting jobs for validation from utils_file import gdir, gfile, get_parent_path f = gfile('/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/PVsynth/training/RES_1mm_tissue/pve_synth_data_92_common_noise_no_gamma/results_cluster', 'model.*tar') d='/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/PVsynth/training/RES_1mm_tissue/' dres = gdir(d,['.*','result']) dresname = get_parent_path(dres,level=2)[1] dresname = [dd.split('_')[0] + '_' + dd.split('_')[1] for dd in dresname] for one_res, resn in zip(dres, dresname): f = gfile(one_res,'model.*tar') fname = get_parent_path(f)[1] for ff in f: print('\"{}\",'.format(ff)) for one_res, resn in zip(dres, dresname): f = gfile(one_res,'model.*tar') fname = get_parent_path(f)[1] for ff in fname: fname_ep = ff.split('_')[1]
def set_model(self, par_model, res_model_file=None, verbose=True, log_filename='training.log'): network_name = par_model['network_name'] losstype = par_model['losstype'] lr = par_model['lr'] in_size = par_model['in_size'] self.cuda = par_model['cuda'] self.max_epochs = par_model['max_epochs'] optim_name = par_model['optim'] if 'optim' in par_model else 'Adam' self.validation_droupout = par_model[ 'validation_droupout'] if 'validation_droupout' in par_model else False if network_name == 'unet_f': self.model = UNet( in_channels=1, dimensions=3, out_classes=1, num_encoding_blocks=3, out_channels_first_layer=16, normalization='batch', padding=True, pooling_type='max', # max avg AdaptiveMax AdaptiveAvg upsampling_type='trilinear', residual=False, dropout=False, monte_carlo_dropout=0.5) elif network_name == 'unet': self.model = SmallUnet(in_channels=1, out_channels=1) elif network_name == 'ConvN': conv_block = par_model['conv_block'] dropout, drop_conv, batch_norm = par_model['dropout'], par_model[ 'drop_conv'], par_model['batch_norm'] linear_block = par_model['linear_block'] output_fnc = par_model[ 'output_fnc'] if 'output_fnc' in par_model else None self.model = ConvN_FC3(in_size=in_size, conv_block=conv_block, linear_block=linear_block, dropout=dropout, drop_conv=drop_conv, batch_norm=batch_norm, output_fnc=output_fnc) network_name += '_C{}_{}_Lin{}_{}_D{}_DC{}'.format( np.abs(conv_block[0]), conv_block[-1], linear_block[0], linear_block[-1], dropout, drop_conv) if output_fnc is not None: network_name += '_fnc_{}'.format(output_fnc) if batch_norm: network_name += '_BN' if self.validation_droupout: network_name += '_VD' self.res_name += '_Size{}_{}_Loss_{}_lr{}'.format( in_size[0], network_name, losstype, lr) if 'Adam' not in optim_name: #only write if not default Adam self.res_name += '_{}'.format(optim_name) self.res_dir += self.res_name + '/' if res_model_file is not None: #to avoid handeling batch size and num worker used for model training self.res_dir, self.res_name = get_parent_path(res_model_file) if not os.path.isdir(self.res_dir): os.mkdir(self.res_dir) self.log = get_log_file(self.res_dir + '/' + log_filename) self.log.info(self.log_string) if losstype == 'MSE': self.loss = tnn.MSELoss() elif losstype == 'L1': self.loss = tnn.L1Loss() elif losstype == 'ssim': self.loss = SSIM3D() elif losstype == 'ssim_dist': self.loss = SSIM3D(distance=2) elif losstype == 'BCE': self.loss = tnn.BCELoss() elif losstype == 'BCElogit': self.loss = tnn.BCEWithLogitsLoss() if self.cuda: self.model = self.model.cuda() self.loss = self.loss.cuda() device = "cuda" else: device = 'cpu' if verbose: self.log.info( summary(self.model, (1, in_size[0], in_size[1], in_size[2]), device=device, batch_size=1)) self.ep_start, self.last_model_saved = load_existing_weights_if_exist( self.res_dir, self.model, log=self.log, device=device, res_model_file=res_model_file) if "Adam" in optim_name: self.optimizer = optim.Adam(self.model.parameters(), lr=lr) elif "SGD" in optim_name: self.optimizer = optim.SGD(self.model.parameters(), lr=lr, momentum=0.5)
import os resdir_mvt = res_dir + '/mvt_param/' resdir_fig = res_dir + '/fig/' try: #on cluster, all job are doing the mkdir at the same time ... if not os.path.isdir(resdir_mvt): os.mkdir(resdir_mvt) if not os.path.isdir(resdir_fig): os.mkdir(resdir_fig) except: pass transfo = get_motion_transform(type=motion_type) torch.manual_seed(seed) np.random.seed(seed) dir_img = get_parent_path([fin])[0] fm = gfile(dir_img, '^mask', {"items": 1}) fp1 = gfile(dir_img, '^p1', {"items": 1}) fp2 = gfile(dir_img, '^p2', {"items": 1}) if len(fm) == 0: #may be in cat12 subdir (like for HCP) fm = gfile(dir_img, '^brain_T1', {"items": 1}) #dir_cat = gdir(dir_img,'cat12') #fm = gfile(dir_cat, '^mask_brain', {"items": 1}) #fp1 = gfile(dir_cat, '^p1', {"items": 1}) #fp2 = gfile(dir_cat, '^p2', {"items": 1}) one_suj = {'image': Image(fin, INTENSITY), 'brain': Image(fm[0], LABEL)} if len(fp1) == 1: one_suj['p1'] = Image(fp1[0], LABEL) if len(fp2) == 1: one_suj['p2'] = Image(fp2[0], LABEL)
def get_dataset_from_option(options): fin = options.image_in dir_sample = options.sample_dir add_affine_zoom, add_affine_rot = options.add_affine_zoom, options.add_affine_rot batch_size, num_workers = options.batch_size, options.num_workers doit = do_training('/tmp/', 'not_use', verbose=True) # adding transformation tc = [] name_suffix = '' #Attention pas de _ dans le name_suffix if options.add_cut_mask > 0: target_shape, mask_key = (182, 218, 182), 'brain' tc = [ CropOrPad(target_shape=target_shape, mask_name=mask_key), ] name_suffix += '_tCropBrain' if add_affine_rot > 0 or add_affine_zoom > 0: if add_affine_zoom == 0: add_affine_zoom = 1 #0 -> no affine so 1 tc.append( RandomAffine(scales=(add_affine_zoom, add_affine_zoom), degrees=(add_affine_rot, add_affine_rot), image_interpolation=Interpolation.NEAREST)) name_suffix += '_tAffineS{}R{}'.format(add_affine_zoom, add_affine_rot) # for hcp should be before RescaleIntensity mask_brain = False if options.add_mask_brain: tc.append(ApplyMask(masking_method='brain')) name_suffix += '_tMaskBrain' mask_brain = True if options.add_rescal_Imax: tc.append(RescaleIntensity(percentiles=(0, 99))) name_suffix += '_tRescale-0-99' if options.add_elastic1: tc.append(get_motion_transform(type='elastic1')) name_suffix += '_tElastic1' if options.add_bias: tc.append(RandomBiasField()) name_suffix += '_tBias' if len(name_suffix) == 0: name_suffix = '_Raw' target = None if len(tc) == 0: tc = None add_to_load, add_to_load_regexp = None, None if len(dir_sample) > 0: print('loading from {}'.format(dir_sample)) if options.add_orig: add_to_load, add_to_load_regexp = 'original', 'notused' data_name = get_parent_path(dir_sample)[1] if mask_brain and 'hcp' in data_name: add_to_load_regexp = 'brain_T' if add_to_load is None: add_to_load = 'brain' else: add_to_load += 'brain' doit.set_data_loader(batch_size=batch_size, num_workers=num_workers, load_from_dir=dir_sample, transforms=tc, add_to_load=add_to_load, add_to_load_regexp=add_to_load_regexp) name_suffix = 'On_' + data_name + name_suffix target = options.target #'ssim' #suppose that if from sample, it should be simulation so set target else: print('working on ') for ff in fin: print(ff) doit.set_data_loader_from_file_list(fin, transforms=tc, batch_size=batch_size, num_workers=num_workers, mask_key=mask_key, mask_regex='^mask') return doit, name_suffix, target
plt.ioff() resdir_fig = res_dir + '/fig/' resdir_mvt = res_dir + '/mvt_param/' fres = res_dir + '/res_motion.csv' if not os.path.exists(fres): print('found no result file {} \n So building it'.format(fres)) print( 'loading {} sample from {} \n estimated time {} h (for 2 iter/s)'. format(len(td), load_from_dir, len(td) / (2 * 60 * 60))) start = time.time() res, extra_info = pd.DataFrame(), dict() for ii, sample in enumerate(tqdm(td)): ff = sample['mvt_csv'] fcsv_name = get_parent_path(ff)[1][0] extra_info = {'mvt_csv': ff, 'sample': fsaved[ii]} res = doit.add_motion_info(sample, res, extra_info) res.to_csv(fres) print('saving {}'.format(fres)) print('done csv in {}'.format((time.time() - start) / 60 / 60)) if do_plot_fig: start = time.time() res = pd.read_csv(fres) param = res.ssim.values nb_val = 30 #96 pind = np.argsort(param) p0 = np.percentile(pind, 10) #first 10 pm1, pm2 = np.percentile(pind, 40), np.percentile(pind, 60)
rlabel = rlabel.sort_index() # alphabetic order labelsujid = rlabel.index rr = rlabell.reindex(rlabel.index).loc[:, ['lesion_PV', 'lesion_WM']] rlabel = pd.concat([rlabel, rr], axis=1, sort=True) # reorder the label as res[0] # rlabel = rlabel.loc[sujid] # ytrue = rlabel.QCOK.values ytrue = rlabel.globalQualitative.values print_accuracy_df(rlabel, ytrue) # prediction mriqc rd = '/network/lustre/dtlake01/opendata/data/ABIDE/mriqc_data/retrain' resfile = gfile(rd, 'data_CATI.*csv$') resname = get_parent_path(resfile, 1)[1] res = [pd.read_csv(f) for f in resfile] for ii, rr in enumerate(res): sujid = [] for ff in rr.subject_id: dd = ff.split('/') if dd[-1] is '': dd.pop() nn = len(dd) sujid.append(dd[nn - 3] + '+' + dd[nn - 2] + '+' + dd[nn - 1]) rr.index = sujid res[ii] = rr.loc[labelsujid] # rr.loc[sujid[::-1]] print_accuracy(res, resname, ytrue,
from torchvision.transforms import Compose import torchio as tio from torchio.data.io import write_image, read_image from utils_file import get_parent_path, gfile, gdir from slices_2 import do_figures_from_file from utils import reduce_name_list, remove_string_from_name_list from utils_plot_results import get_ep_iter_from_res_name, plot_resdf, plot_train_val_results, \ transform_history_to_factor, parse_history import commentjson as json #res_valOn dd = gfile('/network/lustre/dtlake01/opendata/data/ds000030/rrr/CNN_cache_new', '_') dir_fig = '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/NN_regres_motion/figure/motion_regress/eval2/' dir_fig = '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/NN_regres_random_noise/figure2/' data_name_list = get_parent_path(dd)[1] dres_reg_exp, figname = ['Reg.*D0_DC0'], ['noise'] dres_reg_exp, figname = ['.*hcp.*ms', '.*hcp.*T1', 'cati.*ms', 'cati.*T1' ], ['hcp_ms', 'hcp_T1', 'cati_ms', 'cati_T1'] sns.set(style="whitegrid") csv_regex = 'res_valOn_' for rr, fign in zip(dres_reg_exp, figname): dres = gdir(dqc, rr) resname = get_parent_path(dres)[1] resname = remove_string_from_name_list(resname, [ 'RegMotNew_', 'Size182_ConvN_C16_256_Lin40_50_', '_B4', '_Loss_L1_lr0.0001', '_nw0_D0' ]) resname = [fign + '_' + zz for zz in resname]
# parameters root_dir = '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/NN_regres_motion/' prefix = "/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/job/job_eval/" name = 'cati_all_ms_ela1_train200' name = 'cati_all_ms_train_cati_ms' name = 'cati_all_T1_train_cati_T1' model = root_dir + 'RegMotNew_ela1_train200_hcp400_ms_B4_nw0_Size182_ConvN_C16_256_Lin40_50_D0_BN_Loss_L1_lr0.0001/model_ep3_it9500.pt' model = root_dir + 'RegMotNew_ela1_train_cati_ms_B4_nw0_Size182_ConvN_C16_256_Lin40_50_D0_BN_Loss_L1_lr0.0001/model_ep8_it1249.pt' model = root_dir + 'RegMotNew_ela1_train_cati_T1_B4_nw0_Size182_ConvN_C16_256_Lin40_50_D0_BN_Loss_L1_lr0.0001/model_ep8_it1249.pt' split = 10 model_name = get_parent_path(model)[1][:-3] if 'cati_all_ms' in name: res = pd.read_csv( '/home/romain.valabregue/datal/QCcnn/CATI_datasets/cati_cenir_all_ms.csv' ) elif 'cati_all_T1' in name: res = pd.read_csv( '/home/romain.valabregue/datal/QCcnn/CATI_datasets/cati_cenir_all_T1.csv' ) fin_all = res.filename nb_jobs = len(fin_all) // split + 1 jobs = [] for njob in range(0, nb_jobs): fin = fin_all[njob * split:(njob + 1) * split]
from torchio.transforms import RandomMotionFromTimeCourse, RandomAffine, \ CenterCropOrPad, RandomElasticDeformation, RandomElasticDeformation, CropOrPad, RandomNoise from torchio import Image, ImagesDataset, transforms, INTENSITY, LABEL, Interpolation, Subject from utils_file import get_parent_path, gfile, gdir from doit_train import do_training, get_motion_transform from slices_2 import do_figures_from_file from utils import reduce_name_list, get_ep_iter_from_res_name #Explore csv results dqc = [ '/network/lustre/iss01/cenir/analyse/irm/users/romain.valabregue/QCcnn/NN_regres_motion' ] dres = gdir(dqc, 'RegMotNew.*train_hcp400_ms.*0001') dres = gdir(dqc, 'RegMotNew.*hcp400_ms.*B4.*L1.*0001') dres = gdir(dqc, 'R.*') resname = get_parent_path(dres)[1] #sresname = [rr[rr.find('hcp400_')+7: rr.find('hcp400_')+17] for rr in resname ]; sresname[2] += 'le-4' sresname = resname commonstr, sresname = reduce_name_list(sresname) print('common str {}'.format(commonstr)) target = 'ssim' target_scale = 1 #target='random_noise'; target_scale=10 legend_str = [] col = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'w'] for ii, oneres in enumerate(dres): fresV = gfile(oneres, 'res_val') fresT = gfile(oneres, 'train.*csv') is_train = False if len(fresT) == 0 else True
dir_cache = '/network/lustre/dtlake01/opendata/data/ds000030/rrr/CNN_cache_new/' jobs = [] scriptsDir = '/network/lustre/iss01/cenir/software/irm/toolbox_python/romain/torchQC' for data_name_val in name_list_val: dir_sample = '{}/{}/'.format(dir_cache, data_name_val) for saved_model in saved_models: for opt_test in options_test: py_options = '--use_gpu 0 --saved_model {} {} '.format( saved_model, opt_test) model_name = get_parent_path(saved_model)[1] cmd_init = '\n'.join([ "#source /network/lustre/iss01/cenir/software/irm/bin/python_path3.6", "#source activate pytorch1.2", "python " + scriptsDir + "/do_eval_model.py \\", py_options + " \\" ]) job = '\n'.join([cmd_init, ' --sample_dir ' + dir_sample]) jobs.append(job) params = dict() params['output_directory'] = prefix + '/jobs/' + job_id params['scripts_to_copy'] = scriptsDir #+ '/*.py'
dir_sample = options.sample_dir target_shape, mask_key = (182, 218, 182), 'brain' tc = [ CropOrPad(target_shape=target_shape, mask_name=mask_key), ] doit = do_training('/tmp/', 'not_use', verbose=True) if len(dir_sample) > 0: print('loading from {}'.format(dir_sample)) doit.set_data_loader(batch_size=batch_size, num_workers=num_workers, load_from_dir=dir_sample, transforms=tc) name += '_' + get_parent_path(dir_sample)[1] else: print('working on ') for ff in fin: print(ff) doit.set_data_loader_from_file_list(fin, transforms=tc, batch_size=batch_size, num_workers=num_workers, mask_key=mask_key, mask_regex='^mask') out_name = 'eval_num_{:04d}'.format(int(val_number)) subdir = 'eval_{}_{}'.format(name, get_parent_path(saved_model)[1][:-3])