def run(args): _info(args.bhv) _info(args.subnet) # set to regression mode args.mode = 'reg' # use data from all participants args.cutoff = 0.5 ''' get dataframe ''' # all-but-subnetwork (invert_flag) if 'minus' in args.subnet: args.invert_flag = True res_path = (RES_DIR + '/roi_%d_net_%d' % (args.roi, args.net) + '_nw_%s' % (args.subnet) + '_bhv_%s' % (args.bhv) + '_trainsize_%d' % (args.train_size) + '_kfold_%d_k_hidden_%d' % (args.k_fold, args.k_hidden) + '_k_layers_%d_batch_size_%d' % (args.k_layers, args.batch_size) + '_num_epochs_%d_z_%d.pkl' % (args.num_epochs, args.zscore)) if not os.path.isfile(res_path): df, bhv_df = _bhv_reg_df(args) results = {} results['train_mode'] = _train(df, bhv_df, args) results['test_mode'] = _test(df, bhv_df, args) with open(res_path, 'wb') as f: pickle.dump(results, f)
def run(args): _info(args.roi_name) param_grid = {'k_hidden':args.k_hidden,'k_layers':args.k_layers} param_grid = [comb for comb in ParameterGrid(param_grid)] print(len(param_grid)) print(len(args.k_layers)) if len(param_grid) == 1: res_path = (RES_DIR + '/%s_%d_net_%d' %(args.roi_name, args.roi, args.net) + '_trainsize_%d' %(args.train_size) + '_k_hidden_%d' %(args.k_hidden[0]) + '_k_layers_%d_batch_size_%d' %(args.k_layers[0], args.batch_size) + '_num_epochs_%d_z_%d.pkl' %(args.num_epochs, args.zscore)) mod_path = res_path.replace('results','models') mod_path = mod_path.replace('pkl','h5') elif len(param_grid) > 1: res_path = (RES_DIR + '/%s_%d_net_%d' %(args.roi_name, args.roi, args.net) + '_trainsize_%d' %(args.train_size) + '_kfold_%d' %(args.k_fold) + '_batch_size_%d' %(args.batch_size) + '_num_epochs_%d_z_%d_GSCV.pkl' %(args.num_epochs, args.zscore)) ''' get dataframe ''' if not os.path.isfile(res_path): start = time.time() df = _clip_class_df(args) print('data loading time: %.2f seconds' %(time.time()-start)) if len(param_grid) == 1: results, results_prob, model = _test(df,args,param_grid[0]) # save results with open(res_path, 'wb') as f: pickle.dump([results, results_prob], f) # save model if not os.path.exists(os.path.dirname(mod_path)): os.makedirs(os.path.dirname(mod_path),exist_ok=True) model.save(mod_path) elif len(param_grid) > 1: results = {} for mm, params in enumerate(param_grid): print('---') print('model{:02d}'.format(mm) + ': ') print(params) print('---') results['model{:02d}'.format(mm)] = _train(df, args, params) # save grid-search CV results with open(res_path, 'wb') as f: pickle.dump([results, param_grid], f)
def run(args): _info(args.roi_name) # Get all combinations of the parameter grid param_grid = {'k_hidden':args.k_hidden,'k_layers':args.k_layers} param_grid = [comb for comb in ParameterGrid(param_grid)] _info('Number of hyperparameter combinations: '+str(len(param_grid))) _info(args.roi_name) _info(args.input_data) ''' get dataframe ''' if len(param_grid) > 1: res_path = (RES_DIR + '/%s_near_miss_%d_trainsize_%d' %(args.roi_name,args.near_miss,args.train_size) + '_kfold_%d' %(args.k_fold) + '_batch_size_%d' %(args.batch_size) + '_num_epochs_%d_z_%d_GSCV.pkl' %(args.num_epochs, args.zscore)) if not os.path.isfile(res_path): df = _emo_class_df(args) results = {} for mm, params in enumerate(param_grid): print('---') print('model{:02d}'.format(mm) + ': ') print(params) print('---') results['model{:02d}'.format(mm)] = _train(df, args, params) with open(res_path, 'wb') as f: pickle.dump([results, param_grid], f) elif len(param_grid) == 1: res_path = (RES_DIR + '/%s_near_miss_%d_trainsize_%d' %(args.roi_name,args.near_miss,args.train_size) + '_kfold_%d_k_hidden_%d' %(args.k_fold, args.k_hidden[0]) + '_k_layers_%d_batch_size_%d' %(args.k_layers[0], args.batch_size) + '_num_epochs_%d_z_%d.pkl' %(args.num_epochs, args.zscore)) mod_path = res_path.replace('results','models') mod_path = mod_path.replace('pkl','h5') if not os.path.isfile(res_path): df = _emo_class_df(args) results = {} results['train_mode'] = _train(df, args, param_grid[0]) results['test_mode'], _, model = _test(df, args, param_grid[0]) # save results with open(res_path, 'wb') as f: pickle.dump(results, f) # save model if not os.path.exists(os.path.dirname(mod_path)): os.makedirs(os.path.dirname(mod_path),exist_ok=True) model.save(mod_path)
def _saliency(df, args): ''' save gradient with respect to the input to use as proxy for importance ''' then = time.time() _info('save gradients') subject_list = np.unique(df['Subject']) train_list = subject_list[:args.train_size] test_list = subject_list[args.train_size:] k_class = len(np.unique(df['y'])) print('number of unique sequences = %d' % k_class) features = [ii for ii in df.columns if 'feat' in ii] gru_model = keras.models.load_model(args.gru_model_path) gru_model.trainable = False mat = {} for i_class in range(k_class): mat[i_class] = [] for subject in tqdm(test_list): subj_df = df[df['Subject'] == subject] subj_df.reset_index(inplace=True) trials = np.split(subj_df, subj_df[subj_df['timepoint'] == 0].index)[1:] for ii, trial in enumerate(trials): seq = trial[trial['Subject'] == subject][features].values label_seq = trial[trial['Subject'] == subject]['y'].values class_label = label_seq[0] X = [seq] X_padded = tf.keras.preprocessing.sequence.pad_sequences( X, padding="post", dtype='float') gradX = _compute_saliency_maps(gru_model, X_padded, i_class) mat[class_label].append(gradX) for i_class in range(k_class): mat[i_class] = np.stack(mat[i_class], axis=0) #participant x time x vox return mat
def run(args): _info(args.subnet) ''' get dataframe ''' # all-but-subnetwork (invert_flag) if 'minus' in args.subnet: args.invert_flag = True res_path = (RES_DIR + '/%s_%d_net_%d' % (args.roi_name, args.roi, args.net) + '_nw_%s' % (args.subnet) + '_trainsize_%d' % (args.train_size) + '_kdim_%d_batch_size_%d' % (args.k_dim, args.batch_size) + '_num_epochs_%d_z_%d.pkl' % (args.num_epochs, args.zscore)) if not os.path.isfile(res_path): df = _clip_class_df(args) results = _test(df, args) with open(res_path, 'wb') as f: pickle.dump(results, f)
def run(args): # load parcellation file parcel, nw_info = _get_parcel(args.roi, args.net) # use glob to get all files with `ext` ext = '*MSMAll_hp2000_clean.dtseries.nii' files = [ y for x in os.walk(args.input_data) for y in glob(os.path.join(x[0], ext)) ] # get list of participants # ID <=> individual participants = set() for file in files: ID = file.split('/MNINonLinear')[0][-6:] participants.add(ID) participants = np.sort(list(participants)) _info('Number of participants = %d' % len(participants)) data = {} for ii, ID in enumerate(participants): ID_files = [file for file in files if ID in file] ID_files = np.sort(ID_files) # if individual has all 4 runs if len(ID_files) == 4: _info('%s: %d/%d' % (ID, (ii + 1), len(participants))) ID_ts, t = [], [] for path in ID_files: roi_ts = _get_roi_ts(path, parcel, nw_info, args) ID_ts.append(roi_ts) t.append(roi_ts.shape[0]) k_time = np.max(t) ''' ID_ts have different temporal length pad zeros (time x roi x number of runs) ''' save_ts = np.zeros((k_time, args.roi, 4)) for run in range(4): run_ts = ID_ts[run] t = run_ts.shape[0] save_ts[:t, :, run] = run_ts data[ID] = save_ts else: _info('%s not processed' % ID) SAVE_DIR = args.output_data if not os.path.exists(SAVE_DIR): os.makedirs(SAVE_DIR) save_path = (SAVE_DIR + '/data_MOVIE_runs_roi_%d_net_%d_ts.pkl' % (args.roi, args.net)) with open(save_path, 'wb') as f: pickle.dump(data, f)
def run(args): # Get all combinations of the parameter grid _info(args.roi_name) # Get all combinations of the parameter grid param_grid = {'k_hidden': args.k_hidden, 'k_layers': args.k_layers} param_grid = [comb for comb in ParameterGrid(param_grid)] print(len(param_grid)) print(len(args.k_layers)) _info('Number of hyperparameter combinations: ' + str(len(param_grid))) _info(args.roi_name) ''' get dataframe ''' if len(param_grid) == 1: res_path = (RES_DIR + '/%s_near_miss_%d_trainsize_%d' % (args.roi_name, args.near_miss, args.train_size) + '_kfold_%d_k_hidden_%d' % (args.k_fold, args.k_hidden[0]) + '_k_layers_%d_batch_size_%d' % (args.k_layers[0], args.batch_size) + '_num_epochs_%d_z_%d.pkl' % (args.num_epochs, args.zscore)) gru_mod_path = res_path.replace('results', 'models') gru_mod_path = gru_mod_path.replace('pkl', 'h5') gru_model_path = gru_mod_path.replace('saliency', 'gru') args.gru_model_path = gru_model_path elif len(param_grid) > 1: res_path = (RES_DIR + '/%s_near_miss_%d_trainsize_%d' % (args.roi_name, args.near_miss, args.train_size) + '_kfold_%d_k_hidden_%d' % (args.k_fold, args.k_hidden[0]) + '_k_layers_%d_batch_size_%d' % (args.k_layers[0], args.batch_size) + '_num_epochs_%d_z_%d.pkl' % (args.num_epochs, args.zscore)) if not os.path.isfile(res_path): df = _emo_class_df(args) if len(param_grid) == 1: mat = _saliency(df, args) # save results with open(res_path, 'wb') as f: pickle.dump(mat, f) elif len(param_grid) > 1: results = {} for mm, params in enumerate(param_grid): print('---') print('model{:02d}'.format(mm) + ': ') print(params) print('---') results['model{:02d}'.format(mm)] = _train(df, args, params) # save grid-search CV results with open(res_path, 'wb') as f: pickle.dump([results, param_grid], f)
def run(args): # Get all combinations of the parameter grid _info(args.roi_name) # Get all combinations of the parameter grid param_grid = {'k_hidden': args.k_hidden, 'k_layers': args.k_layers} param_grid = [comb for comb in ParameterGrid(param_grid)] print(len(param_grid)) print(len(args.k_layers)) _info('Number of hyperparameter combinations: ' + str(len(param_grid))) _info(args.roi_name) ''' get dataframe ''' if len(param_grid) == 1: res_path = (RES_DIR + '/%s_%d_net_%d' % (args.roi_name, args.roi, args.net) + '_trainsize_%d' % (args.train_size) + '_k_hidden_%d' % (args.k_hidden[0]) + '_k_layers_%d_batch_size_%d' % (args.k_layers[0], args.batch_size) + '_num_epochs_%d_z_%d.pkl' % (args.num_epochs, args.zscore)) gru_mod_path = res_path.replace('results', 'models') gru_mod_path = gru_mod_path.replace('pkl', 'h5') gru_model_path = gru_mod_path.replace('null_saliency', 'gru') args.gru_model_path = gru_model_path elif len(param_grid) > 1: res_path = (RES_DIR + '/%s_%d_net_%d' % (args.roi_name, args.roi, args.net) + '_trainsize_%d' % (args.train_size) + '_kfold_%d' % (args.k_fold) + '_batch_size_%d' % (args.batch_size) + '_num_epochs_%d_z_%d_GSCV.pkl' % (args.num_epochs, args.zscore)) if not os.path.isfile(res_path): start = time.clock() #df = _clip_class_df(args) with open('data/df.pkl', 'rb') as f: df = pickle.load(f) print('data loading time: %.2f seconds' % (time.clock() - start)) if len(param_grid) == 1: _saliency(df, args) elif len(param_grid) > 1: results = {} for mm, params in enumerate(param_grid): print('---') print('model{:02d}'.format(mm) + ': ') print(params) print('---') results['model{:02d}'.format(mm)] = _train(df, args, params) # save grid-search CV results with open(res_path, 'wb') as f: pickle.dump([results, param_grid], f)
def run(args): _info(args.bhv) _info(args.subnet) # set to regression mode args.mode = 'reg' ''' get dataframe ''' # all-but-subnetwork (invert_flag) if 'minus' in args.subnet: args.invert_flag = True res_path = (RES_DIR + '/roi_%d_net_%d' % (args.roi, args.net) + '_nw_%s' % (args.subnet) + '_bhv_%s_cutoff_%0.1f' % (args.bhv, args.cutoff) + '_corrthresh_%0.1f' % (args.corr_thresh) + '_kfold_%d_z_%d.pkl' % (args.k_fold, args.zscore)) if not os.path.isfile(res_path): df, bhv_df = _bhv_reg_df(args) results = _train(df, bhv_df, args) with open(res_path, 'wb') as f: pickle.dump(results, f)
def _saliency(df, args): ''' save gradient with respect to the input to use as proxy for importance ''' then = time.time() _info('save gradients') subject_list = np.unique(df['Subject']) k_class = len(np.unique(df['y'])) print('number of unique sequences = %d' % k_class) # create columns for gradients # don't use number of ROI in case of subnetwork features = [ii for ii in df.columns if 'feat' in ii] grads = ['grad_%d' % ii for ii in range(len(features))] for grad in grads: df.loc[:, grad] = np.nan gru_model = keras.models.load_model(args.gru_model_path) gru_model.trainable = False for i_class in tqdm(range(k_class)): for subject in subject_list: if i_class == 0: # must handle test retest differently seqs = df[(df['Subject'] == subject) & (df['y'] == 0)][features].values gradX = np.zeros(seqs.shape) k_time = int(seqs.shape[0] / K_RUNS) for i_run in range(K_RUNS): seq = seqs[i_run * k_time:(i_run + 1) * k_time, :] if args.zscore: # zscore each seq that goes into model seq = (1 / np.std(seq)) * (seq - np.mean(seq)) X = [seq] X_padded = tf.keras.preprocessing.sequence.pad_sequences( X, padding="post", dtype='float') gX = _compute_saliency_maps(gru_model, X_padded, i_class) gradX[i_run * k_time:(i_run + 1) * k_time, :] = gX else: seq = df[(df['Subject'] == subject) & (df['y'] == i_class)][features].values if args.zscore: # zscore each seq that goes into model seq = (1 / np.std(seq)) * (seq - np.mean(seq)) X = [seq] X_padded = tf.keras.preprocessing.sequence.pad_sequences( X, padding="post", dtype='float') gradX = _compute_saliency_maps(gru_model, X_padded, i_class) df.loc[(df['Subject'] == subject) & (df['y'] == i_class), grads] = gradX.squeeze() sal_df = df[['Subject', 'timepoint', 'y'] + grads] return sal_df
def _test(df, args): ''' test subject results view only for best cross-val parameters ''' _info('test mode') # get X-y from df subject_list = df['Subject'].unique() train_list = subject_list[:args.train_size] test_list = subject_list[args.train_size:] features = [ii for ii in df.columns if 'feat' in ii] k_feat = len(features) print('number of classes = %d' % (args.k_class)) # length of each clip clip_time = np.zeros(args.k_class) for ii in range(args.k_class): class_df = df[df['y'] == ii] clip_time[ii] = np.max(np.unique(class_df['timepoint'])) + 1 clip_time = clip_time.astype(int) # df saves float _info('seq lengths = %s' % clip_time) # results dict init results = {} # mean accuracy across time results['train'] = np.zeros(len(test_list)) results['val'] = np.zeros(len(test_list)) # per class temporal accuracy results['t_train'] = {} results['t_test'] = {} for ii in range(args.k_class): results['t_train'][ii] = np.zeros((len(test_list), clip_time[ii])) results['t_test'][ii] = np.zeros((len(test_list), clip_time[ii])) ''' init model ''' model = TCNClassifier(k_feat, args.k_hidden, args.k_wind, args.k_class) model.to(args.device) print(model) lossfn = nn.CrossEntropyLoss(ignore_index=-100) # if input is cuda, loss function is auto cuda opt = torch.optim.Adam(model.parameters()) # get train, val sequences X_train, train_len, y_train = _get_seq(df, train_list, args) X_test, test_len, y_test = _get_seq(df, test_list, args) max_length = torch.max(train_len) ''' train classifier ''' permutation = torch.randperm(X_train.size()[0]) losses = np.zeros(args.num_epochs) # then = time.time() for epoch in range(args.num_epochs): for i in range(0, X_train.size()[0], args.batch_size): indices = permutation[i:i + args.batch_size] batch_x, batch_y = X_train[indices], y_train[indices] y_pred = model(batch_x) loss = lossfn(y_pred.view(-1, args.k_class), batch_y.view(-1)) opt.zero_grad() loss.backward() opt.step() losses[epoch] = loss _info(losses) # print('--- train time = %0.4f seconds ---' % (time.time() - then)) ''' results on train data ''' a, a_t, c_mtx = _ff_test_acc(model, X_train, y_train, train_len, max_length, clip_time, len(train_list)) results['train'] = a print('tacc = %0.3f' % np.mean(a)) for ii in range(args.k_class): results['t_train'][ii] = a_t[ii] results['train_conf_mtx'] = c_mtx ''' results on test data ''' a, a_t, c_mtx = _ff_test_acc(model, X_test, y_test, test_len, max_length, clip_time, len(test_list)) results['test'] = a print('sacc = %0.3f' % np.mean(a)) for ii in range(args.k_class): results['t_test'][ii] = a_t[ii] results['test_conf_mtx'] = c_mtx return results
def _train(df, args): ''' cross-validation results ''' # set pytorch device torch.manual_seed(K_SEED) use_cuda = torch.cuda.is_available() args.device = torch.device('cuda:0' if use_cuda else 'cpu') if use_cuda: _info('cuda') else: _info('cpu') # get X-y from df subject_list = df['Subject'].unique() train_list = subject_list[:args.train_size] test_list = subject_list[args.train_size:] print('number of subjects = %d' % (len(subject_list))) features = [ii for ii in df.columns if 'feat' in ii] k_feat = len(features) print('number of features = %d' % (k_feat)) args.k_class = len(np.unique(df['y'])) print('number of classes = %d' % (args.k_class)) # length of each clip clip_time = np.zeros(args.k_class) for ii in range(args.k_class): class_df = df[df['y'] == ii] clip_time[ii] = np.max(np.unique(class_df['timepoint'])) + 1 clip_time = clip_time.astype(int) # df saves float _info('seq lengths = %s' % clip_time) # results dict init results = {} # mean accuracy across time results['train'] = np.zeros(args.k_fold) results['val'] = np.zeros(args.k_fold) # confusion matrices results['train_conf_mtx'] = np.zeros((args.k_class, args.k_class)) results['val_conf_mtx'] = np.zeros((args.k_class, args.k_class)) # per class temporal accuracy results['t_train'] = {} results['t_val'] = {} for ii in range(args.k_class): results['t_train'][ii] = np.zeros((args.k_fold, clip_time[ii])) results['t_val'][ii] = np.zeros((args.k_fold, clip_time[ii])) i_fold = 0 kf = KFold(n_splits=args.k_fold, random_state=K_SEED) for train, val in kf.split(train_list): _info('fold: %d/%d' % (i_fold + 1, args.k_fold)) # ***between-subject train-val split train_subs = [train_list[ii] for ii in train] val_subs = [train_list[ii] for ii in val] ''' init model ''' model = TCNClassifier(k_feat, args.k_hidden, args.k_wind, args.k_class) model.to(args.device) print(model) lossfn = nn.CrossEntropyLoss(ignore_index=-100) # if input is cuda, loss function is auto cuda opt = torch.optim.Adam(model.parameters()) # get train, val sequences X_train, train_len, y_train = _get_seq(df, train_subs, args) X_val, val_len, y_val = _get_seq(df, val_subs, args) max_length = torch.max(train_len) ''' train classifier ''' permutation = torch.randperm(X_train.size()[0]) losses = np.zeros(args.num_epochs) # then = time.time() for epoch in range(args.num_epochs): for i in range(0, X_train.size()[0], args.batch_size): indices = permutation[i:i + args.batch_size] batch_x, batch_y = X_train[indices], y_train[indices] y_pred = model(batch_x) loss = lossfn(y_pred.view(-1, args.k_class), batch_y.view(-1)) opt.zero_grad() loss.backward() opt.step() losses[epoch] = loss _info(losses) # print('--- train time = %0.4f seconds ---' % (time.time() - then)) ''' results on train data ''' a, a_t, c_mtx = _ff_acc(model, X_train, y_train, train_len, max_length, clip_time) results['train'][i_fold] = a print('tacc = %0.3f' % a) for ii in range(args.k_class): results['t_train'][ii][i_fold] = a_t[ii] results['train_conf_mtx'] += c_mtx ''' results on val data ''' a, a_t, c_mtx = _ff_acc(model, X_val, y_val, val_len, max_length, clip_time) results['val'][i_fold] = a print('vacc = %0.3f' % a) for ii in range(args.k_class): results['t_val'][ii][i_fold] = a_t[ii] results['val_conf_mtx'] += c_mtx i_fold += 1 return results
def _train(df, args, params): ''' cross-validation results ''' _info('train mode: Running grid search') # get X-y from df subject_list = df['Subject'].unique() train_list = subject_list[:args.train_size] test_list = subject_list[args.train_size:] print('number of subjects = %d' % (len(subject_list))) features = [ii for ii in df.columns if 'feat' in ii] k_feat = len(features) print('number of features = %d' % (k_feat)) args.k_class = len(np.unique(df['y'])) print('number of classes = %d' % (args.k_class)) # length of each clip clip_time = np.zeros(args.k_class) for ii in range(args.k_class): class_df = df[df['y'] == ii] clip_time[ii] = np.max(np.unique(class_df['timepoint'])) + 1 clip_time = clip_time.astype(int) # df saves float print('seq lengths = %s' % clip_time) # results dict init results = {} # mean accuracy across time results['train'] = np.zeros(args.k_fold) results['val'] = np.zeros(args.k_fold) # confusion matrices results['train_conf_mtx'] = np.zeros((args.k_class, args.k_class)) results['val_conf_mtx'] = np.zeros((args.k_class, args.k_class)) # per class temporal accuracy results['t_train'] = {} results['t_val'] = {} for ii in range(args.k_class): results['t_train'][ii] = np.zeros((args.k_fold, clip_time[ii])) results['t_val'][ii] = np.zeros((args.k_fold, clip_time[ii])) i_fold = 0 kf = KFold(n_splits=args.k_fold, random_state=K_SEED) for train, val in kf.split(train_list): _info('fold: %d/%d' % (i_fold + 1, args.k_fold)) # ***between-subject train-val split train_subs = [train_list[ii] for ii in train] val_subs = [train_list[ii] for ii in val] # get train, val sequences X_train, train_len, y_train = _get_seq(df, train_subs, args) X_val, val_len, y_val = _get_seq(df, val_subs, args) ''' train classifier ''' then = time.time() model = TCNClassifier(X_train, k_hidden=params['k_hidden'], k_wind=params['k_wind'], k_class=args.k_class) model.fit(X_train, y_train, epochs=args.num_epochs, validation_split=0.2, batch_size=args.batch_size, verbose=1) print('--- train time = %0.4f seconds ---' % (time.time() - then)) trainable = np.sum([ tf.reshape(params, -1).shape[0] for params in model.trainable_variables ]) print('Total trainable parameters: %i' % trainable) ''' results on train data ''' a, a_t, c_mtx = _ff_acc(model, X_train, y_train, clip_time) results['train'][i_fold] = a print('tacc = %0.3f' % a) for ii in range(args.k_class): results['t_train'][ii][i_fold] = a_t[ii] results['train_conf_mtx'] += c_mtx ''' results on val data ''' a, a_t, c_mtx = _ff_acc(model, X_val, y_val, clip_time) results['val'][i_fold] = a print('vacc = %0.3f' % a) for ii in range(args.k_class): results['t_val'][ii][i_fold] = a_t[ii] results['val_conf_mtx'] += c_mtx i_fold += 1 return results
def _test(df, args, params): ''' test subject results view only for best cross-val parameters ''' _info('test mode') # get X-y from df subject_list = df['Subject'].unique() train_list = subject_list[:args.train_size] test_list = subject_list[args.train_size:] print('number of subjects = %d' % (len(subject_list))) features = [ii for ii in df.columns if 'feat' in ii] k_feat = len(features) print('number of features = %d' % (k_feat)) args.k_class = len(np.unique(df['y'])) print('number of classes = %d' % (args.k_class)) # length of each clip clip_time = np.zeros(args.k_class) for ii in range(args.k_class): class_df = df[df['y'] == ii] clip_time[ii] = np.max(np.unique(class_df['timepoint'])) + 1 clip_time = clip_time.astype(int) # df saves float print('seq lengths = %s' % clip_time) ''' init model ''' # get train, test sequences X_train, train_len, y_train = _get_seq(df, train_list, args) X_test, test_len, y_test = _get_seq(df, test_list, args) max_length = tf.math.reduce_max(train_len).numpy() ''' train encoder ''' then = time.time() model_encoder = GRUEncoder(X_train, args.gru_model_path, k_layers=params['k_layers'], k_hidden=params['k_hidden'], k_dim=args.k_dim, k_class=args.k_class) model_encoder.fit(X_train, y_train, epochs=args.num_epochs, validation_split=0.2, batch_size=args.batch_size, verbose=1) ''' encoder results ''' results = {} a, a_t, c_mtx = _gru_test_acc(model_encoder, X_train, y_train, clip_time, len(train_list)) results['train'] = a a, a_t, c_mtx = _gru_test_acc(model_encoder, X_test, y_test, clip_time, len(test_list)) results['test'] = a ''' get encoder trajectories ''' traj_train = _gruenc_test_traj(model_encoder, X_train) traj_test = _gruenc_test_traj(model_encoder, X_test) ''' apply mask on trajectories ''' mask = X_train[:, :, 0] == 0.0 traj_train[mask, :] = 0.0 mask = X_test[:, :, 0] == 0.0 traj_test[mask, :] = 0.0 ''' train decoder ''' model_decoder = GRUDecoder(traj_train, X_train, k_layers=args.k_layers[0], lr=0.001) model_decoder.fit(traj_train, X_train, epochs=args.num_epochs, validation_split=0.2, batch_size=args.batch_size, verbose=1) ''' evaluate decoder ''' train_mask = X_train != 0 test_mask = X_test != 0 ''' results on train data ''' outputs = model_decoder.predict(traj_train) o = outputs[train_mask == True] y = X_train[train_mask == True] a = mean_squared_error(o, y) print('train_recon mse = %0.3f' % a) results['train_mse'] = a a = r2_score(o, y) results['train_r2'] = a print('train_recon r2 = %0.3f' % a) ''' results on test data ''' outputs = model_decoder.predict(traj_test) o = outputs[test_mask == True] y = X_test[test_mask == True] a = mean_squared_error(o, y) print('test_recon mse = %0.3f' % a) results['test_mse'] = a a = r2_score(o, y) results['test_r2'] = a print('test_recon r2 = %0.3f' % a) ''' compare to pca reconstruction ''' train_mse, test_mse, train_r2, test_r2, pca_var = _get_pca_recon( df, train_list, test_list, args) results['pca_var'] = pca_var ''' results on train data ''' results['train_pca_mse'] = train_mse results['train_pca_r2'] = train_r2 print('t_pca_recon r2 = %0.3f' % train_r2) ''' results on test data ''' results['test_pca_mse'] = test_mse results['test_pca_r2'] = test_r2 print('s_pca_recon r2 = %0.3f' % test_r2) return results
def _train(df, bhv_df, args): # get X-y from df feature = [ii for ii in df.columns if 'feat' in ii] k_feat = len(feature) print('number of features = %d' % (k_feat)) k_clip = len(np.unique(df['c'])) print('number of clip = %d' % (k_clip)) subject_list = bhv_df['Subject'].unique() train_list = subject_list[:args.train_size] test_list = subject_list[args.train_size:] # length of each clip clip_time = np.zeros(k_clip) for ii in range(k_clip): class_df = df[df['c'] == ii] clip_time[ii] = np.max(np.unique(class_df['timepoint'])) + 1 clip_time = clip_time.astype(int) # df saves float _info('seq lengths = %s' % clip_time) # init dict for all results results = {} # true and predicted scores and clip label results['y'] = {} results['y_hat'] = {} results['c'] = {} for score in SCORES: # mean scores across time results['train_%s' % score] = np.zeros(args.k_fold) results['val_%s' % score] = np.zeros(args.k_fold) # per clip temporal score results['t_train_%s' % score] = {} results['t_val_%s' % score] = {} for ii in range(k_clip): results['t_train_%s' % score][ii] = np.zeros( (args.k_fold, clip_time[ii])) results['t_val_%s' % score][ii] = np.zeros( (args.k_fold, clip_time[ii])) kf = KFold(n_splits=args.k_fold, random_state=K_SEED) # get participant lists for each assigned class # ensure they're only in train_list class_list = {} for ii in range(args.k_class): class_list[ii] = bhv_df[(bhv_df['Subject'].isin(train_list)) & (bhv_df['y'] == ii)]['Subject'].values print('No. of participants in class {} = {}'.format( ii, len(class_list[ii]))) ''' split participants in each class with kf nearly identical ratio of train and val, in all classes ''' split = {} for ii in range(args.k_class): split[ii] = kf.split(class_list[ii]) for i_fold in range(args.k_fold): _info('fold: %d/%d' % (i_fold + 1, args.k_fold)) # ***between-subject train-val split train_subs, val_subs = [], [] for ii in range(args.k_class): train, val = next(split[ii]) for jj in train: train_subs.append(class_list[ii][jj]) for jj in val: val_subs.append(class_list[ii][jj]) ''' model main ''' X_train, train_len, y_train, c_train = _get_seq(df, train_subs, args) X_val, val_len, y_val, c_val = _get_seq(df, val_subs, args) max_length = tf.reduce_max(train_len) ''' train regression model ''' then = time.time() model = GRURegressor(X_train, k_hidden=args.k_hidden, k_layers=args.k_layers, l2=args.l2, dropout=args.dropout, lr=args.lr) model.fit(X_train, y_train.reshape(y_train.shape[0], y_train.shape[1], 1), batch_size=args.batch_size, epochs=args.num_epochs, verbose=1, validation_split=0.2) print('--- train time = %0.4f seconds ---' % (time.time() - then)) ''' results on train data ''' s, s_t, _, _, _ = dnn_score(model, X_train, y_train, c_train, train_len, max_length, clip_time, model_type=model_type) for score in SCORES: results['train_%s' % score][i_fold] = s[score] for ii in range(k_clip): results['t_train_%s' % score][ii][i_fold] = s_t[ii][score] print('train p = %0.3f' % s['p']) ''' results on val data ''' s, s_t, y, y_hat, c = dnn_score(model, X_val, y_val, c_val, val_len, max_length, clip_time, model_type=model_type) for score in SCORES: results['val_%s' % score][i_fold] = s[score] for ii in range(k_clip): results['t_val_%s' % score][ii][i_fold] = s_t[ii][score] print('val p = %0.3f' % s['p']) results['y'][i_fold] = y results['y_hat'][i_fold] = y_hat results['c'][i_fold] = c return results
def _test(df, args, params): ''' test subject results view only for best cross-val parameters ''' _info('test mode') # get X-y from df subject_list = df['Subject'].unique() train_list = subject_list[:args.train_size] test_list = subject_list[args.train_size:] print('number of subjects = %d' % (len(subject_list))) features = [ii for ii in df.columns if 'feat' in ii] k_feat = len(features) print('number of features = %d' % (k_feat)) args.k_class = len(np.unique(df['y'])) print('number of classes = %d' % (args.k_class)) # length of each clip clip_time = np.zeros(args.k_class) for ii in range(args.k_class): class_df = df[df['y'] == ii] clip_time[ii] = np.max(np.unique(class_df['timepoint'])) + 1 clip_time = clip_time.astype(int) # df saves float print('seq lengths = %s' % clip_time) # results dict init results = {} # mean accuracy across time results['train'] = np.zeros(len(test_list)) results['val'] = np.zeros(len(test_list)) # per class temporal accuracy results['t_train'] = {} results['t_test'] = {} for ii in range(args.k_class): results['t_train'][ii] = np.zeros((len(test_list), clip_time[ii])) results['t_test'][ii] = np.zeros((len(test_list), clip_time[ii])) results_prob = {} for method in 'train test'.split(): results_prob[method] = {} for measure in 'acc t_prob'.split(): results_prob[method][measure] = {} ''' init model ''' # get train, test sequences X_train, train_len, y_train = _get_seq(df, train_list, args) X_test, test_len, y_test = _get_seq(df, test_list, args) ''' train classifier ''' then = time.time() model = TCNClassifier(X_train, k_hidden=params['k_hidden'], k_wind=params['k_wind'], k_class=args.k_class) model.fit(X_train, y_train, epochs=args.num_epochs, validation_split=0.2, batch_size=args.batch_size, verbose=1) print('--- train time = %0.4f seconds ---' % (time.time() - then)) ''' results on train data ''' a, a_t, c_mtx = _ff_test_acc(model, X_train, y_train, clip_time, len(train_list)) results['train'] = a print('tacc = %0.3f' % np.mean(a)) for ii in range(args.k_class): results['t_train'][ii] = a_t[ii] results['train_conf_mtx'] = c_mtx # train temporal probs results_prob['train']['acc'] = model.evaluate(X_train, y_train)[1] X_train_probs = model.predict(X_train) results_prob['train']['t_prob'] = _get_true_class_prob( y_train, X_train_probs, train_len) ''' results on test data ''' a, a_t, c_mtx = _ff_test_acc(model, X_test, y_test, clip_time, len(test_list)) results['test'] = a print('sacc = %0.3f' % np.mean(a)) for ii in range(args.k_class): results['t_test'][ii] = a_t[ii] results['test_conf_mtx'] = c_mtx # test temporal probs results_prob['test']['acc'] = model.evaluate(X_test, y_test)[1] X_test_probs = model.predict(X_test) results_prob['test']['t_prob'] = _get_true_class_prob( y_test, X_test_probs, test_len) return results, results_prob, model
def _test(df, bhv_df, args): _info('test mode') # get X-y from df features = [ii for ii in df.columns if 'feat' in ii] k_feat = len(features) print('number of features = %d' % (k_feat)) k_clip = len(np.unique(df['c'])) print('number of clips = %d' % (k_clip)) subject_list = bhv_df['Subject'].unique() train_list = subject_list[:args.train_size] test_list = subject_list[args.train_size:] # length of each clip clip_time = np.zeros(k_clip) for ii in range(k_clip): class_df = df[df['c'] == ii] clip_time[ii] = np.max(np.unique(class_df['timepoint'])) + 1 clip_time = clip_time.astype(int) # df saves float _info('seq lengths = %s' % clip_time) # init dict for all results results = {} for score in SCORES: # per clip temporal score results['t_train_%s' % score] = {} results['t_test_%s' % score] = {} for ii in range(k_clip): results['t_train_%s' % score][ii] = np.zeros(clip_time[ii]) results['t_test_%s' % score][ii] = np.zeros(clip_time[ii]) ''' model main ''' # get train, test sequences X_train, train_len, y_train, c_train = _get_seq(df, train_list, args) X_test, test_len, y_test, c_test = _get_seq(df, test_list, args) max_length = tf.reduce_max(train_len) ''' test regression model ''' then = time.time() model = GRURegressor(X_train, k_hidden=args.k_hidden, k_layers=args.k_layers, l2=args.l2, dropout=args.dropout, lr=args.lr) model.fit(X_train, y_train.reshape(y_train.shape[0], y_train.shape[1], 1), batch_size=args.batch_size, epochs=args.num_epochs, verbose=1, validation_split=0.2) print('--- train time = %0.4f seconds ---' % (time.time() - then)) ''' results on train data ''' s, s_t, _, _, _ = dnn_score(model, X_train, y_train, c_train, train_len, max_length, clip_time, model_type=model_type) for score in SCORES: results['train_%s' % score] = s[score] for ii in range(k_clip): results['t_train_%s' % score][ii] = s_t[ii][score] print('train p = %0.3f' % s['p']) ''' results on test data ''' s, s_t, y, y_hat, c = dnn_score(model, X_test, y_test, c_test, test_len, max_length, clip_time, model_type=model_type) for score in SCORES: results['test_%s' % score] = s[score] for ii in range(k_clip): results['t_test_%s' % score][ii] = s_t[ii][score] print('test p = %0.3f' % s['p']) results['y'] = y results['y_hat'] = y_hat results['c'] = c return results
def run(args): # Get all combinations of the parameter grid _info(args.roi_name) # Get all combinations of the parameter grid param_grid = {'k_hidden': args.k_hidden, 'k_layers': args.k_layers} param_grid = [comb for comb in ParameterGrid(param_grid)] print(len(param_grid)) print(len(args.k_layers)) _info('Number of hyperparameter combinations: ' + str(len(param_grid))) _info(args.roi_name) ''' get dataframe ''' if len(param_grid) == 1: res_path = (RES_DIR + '/%s_%d_net_%d' % (args.roi_name, args.roi, args.net) + '_trainsize_%d' % (args.train_size) + '_k_hidden_%d' % (args.k_hidden[0]) + '_kdim_%d' % (args.k_dim) + '_k_layers_%d_batch_size_%d' % (args.k_layers[0], args.batch_size) + '_num_epochs_%d_z_%d.pkl' % (args.num_epochs, args.zscore)) mod_path = res_path.replace('results', 'models') mod_path = mod_path.replace('pkl', 'h5') gru_model_path = mod_path.replace('gruencoder', 'gru') gru_model_path = gru_model_path.replace('_kdim_%d' % (args.k_dim), '') args.gru_model_path = gru_model_path elif len(param_grid) > 1: res_path = (RES_DIR + '/%s_%d_net_%d' % (args.roi_name, args.roi, args.net) + '_trainsize_%d' % (args.train_size) + '_kfold_%d' % (args.k_fold) + '_batch_size_%d' % (args.batch_size) + '_num_epochs_%d_z_%d_GSCV.pkl' % (args.num_epochs, args.zscore)) if not os.path.isfile(res_path): start = time.clock() df = _clip_class_df(args) print('data loading time: %.2f seconds' % (time.clock() - start)) if len(param_grid) == 1: results, results_prob, model = _test(df, args, param_grid[0]) print('TESTING DONE :)') # forward pass data and save encodings #traj_df = {} #for run in range(K_RUNS): # df = _clip_class_rest_df(args, run) # traj_df[run] = _get_trajectories(df, model, args) # save results with open(res_path, 'wb') as f: pickle.dump([results, results_prob], f) # save model #model.save(mod_path) elif len(param_grid) > 1: results = {} for mm, params in enumerate(param_grid): print('---') print('model{:02d}'.format(mm) + ': ') print(params) print('---') results['model{:02d}'.format(mm)] = _train(df, args, params) # save grid-search CV results with open(res_path, 'wb') as f: pickle.dump([results, param_grid], f)
def _saliency(df, args): ''' save gradient with respect to the input to use as proxy for importance ''' then = time.time() _info('save gradients') subject_list = np.unique(df['Subject']) k_class = len(np.unique(df['y'])) print('number of unique sequences = %d' % k_class) # create columns for gradients # don't use number of ROI in case of subnetwork features = [ii for ii in df.columns if 'feat' in ii] grads = ['grad_%d' % ii for ii in range(len(features))] for grad in grads: df.loc[:, grad] = np.nan gru_model = keras.models.load_model(args.gru_model_path) gru_model.trainable = False # length of each clip clip_time = np.zeros(k_class, dtype=np.int) for i_clip in range(k_class): class_df = df[df['y'] == i_clip] clip_time[i_clip] = int(np.max(np.unique(class_df['timepoint'])) + 1) num_perms = 100 for idx_perm in tqdm(range(1, num_perms)): """ shuffle labels """ class_labels = [] for i_class in range(k_class): if i_class == 0: class_labels += [0] * len(subject_list) else: class_labels += [i_class] * len(subject_list) random.shuffle(class_labels) """""" counter = 0 for i_class in range(k_class): if i_class != 14: continue for subject in subject_list: class_label = class_labels[counter] counter += 1 if i_class == 0: # must handle test retest differently seqs = df[(df['Subject'] == subject) & (df['y'] == 0)][features].values gradX = np.zeros(seqs.shape) k_time = int(seqs.shape[0] / K_RUNS) for i_run in range(K_RUNS): seq = seqs[i_run * k_time:(i_run + 1) * k_time, :] if args.zscore: # zscore each seq that goes into model seq = (1 / np.std(seq)) * (seq - np.mean(seq)) X = [seq] X_padded = tf.keras.preprocessing.sequence.pad_sequences( X, padding="post", dtype='float') gX = _compute_saliency_maps(gru_model, X_padded, i_class) gradX[i_run * k_time:(i_run + 1) * k_time, :] = gX else: seq = df[(df['Subject'] == subject) & (df['y'] == class_label)][features].values if args.zscore: # zscore each seq that goes into model seq = (1 / np.std(seq)) * (seq - np.mean(seq)) X = [seq] X_padded = tf.keras.preprocessing.sequence.pad_sequences( X, padding="post", dtype='float') gradX = _compute_saliency_maps(gru_model, X_padded, i_class) gradX = gradX.squeeze() if clip_time[i_class] > clip_time[class_label]: gradX = np.vstack([ gradX, np.zeros( (clip_time[i_class] - clip_time[class_label], gradX.shape[1])) ]) df.loc[(df['Subject'] == subject) & (df['y'] == i_class), grads] = gradX[0:clip_time[i_class], :] sal_df = df[['Subject', 'timepoint', 'y'] + grads] with open( "/home/joyneelm/movie-clip-predictions/results/null_saliency/label_shuffle_starwars/perm_{:03}.pkl" .format(idx_perm), 'wb') as f: pickle.dump(sal_df, f) return
def _test(df, args): ''' test subject results view only for best cross-val parameters ''' _info('test mode') # get X-y from df subject_list = df['Subject'].unique() train_list = subject_list[:args.train_size] test_list = subject_list[args.train_size:] pc_df = _get_pc(df, train_list, test_list, args) print('number of subjects = %d' % (len(subject_list))) features = [ii for ii in df.columns if 'feat' in ii] k_feat = len(features) print('number of features = %d' % (k_feat)) args.k_class = len(np.unique(df['y'])) print('number of classes = %d' % (args.k_class)) # length of each clip clip_time = np.zeros(args.k_class) for ii in range(args.k_class): class_df = df[df['y'] == ii] clip_time[ii] = np.max(np.unique(class_df['timepoint'])) + 1 clip_time = clip_time.astype(int) # df saves float _info('seq lengths = %s' % clip_time) # results dict init results = {} # mean accuracy across time results['train'] = np.zeros(len(test_list)) results['val'] = np.zeros(len(test_list)) # per class temporal accuracy results['t_train'] = {} results['t_test'] = {} for ii in range(args.k_class): results['t_train'][ii] = np.zeros((len(test_list), clip_time[ii])) results['t_test'][ii] = np.zeros((len(test_list), clip_time[ii])) ''' init model ''' # get train, test sequences X_train, train_len, y_train = _get_seq(pc_df, train_list, args) X_test, test_len, y_test = _get_seq(pc_df, test_list, args) ''' train classifier ''' then = time.time() model = LogReg(k_dim=args.k_dim, k_class=args.k_class) model.fit(X_train, y_train, epochs=args.num_epochs, validation_split=0.2, batch_size=args.batch_size, verbose=1) print('--- train time = %0.4f seconds ---' % (time.time() - then)) ''' results on train data ff_test_acc works for logreg ''' a, a_t, c_mtx = _ff_test_acc(model, X_train, y_train, clip_time, len(train_list)) results['train'] = a print('tacc = %0.3f' % np.mean(a)) for ii in range(args.k_class): results['t_train'][ii] = a_t[ii] ''' results on test data ''' a, a_t, c_mtx = _ff_test_acc(model, X_test, y_test, clip_time, len(test_list)) results['test'] = a print('sacc = %0.3f' % np.mean(a)) for ii in range(args.k_class): results['t_test'][ii] = a_t[ii] return results
def _train(df, bhv_df, args): # get X-y from df features = [ii for ii in df.columns if 'feat' in ii] k_feat = len(features) print('number of features = %d' % (k_feat)) k_clip = len(np.unique(df['c'])) print('number of clips = %d' % (k_clip)) subject_list = bhv_df['Subject'].unique() train_list = subject_list[:args.train_size] test_list = subject_list[args.train_size:] # init dict for all results results = {} # true and predicted scores and clip label results['y'] = {} results['y_hat'] = {} results['c'] = {} for score in SCORES: # mean scores across time results['train_%s' % score] = np.zeros(args.k_fold) results['val_%s' % score] = np.zeros(args.k_fold) # per clip score results['c_train_%s' % score] = {} results['c_val_%s' % score] = {} for ii in range(k_clip): results['c_train_%s' % score][ii] = np.zeros(args.k_fold) results['c_val_%s' % score][ii] = np.zeros(args.k_fold) kf = KFold(n_splits=args.k_fold, random_state=K_SEED) # get participant lists for each assigned class class_list = {} for ii in range(args.k_class): class_list[ii] = bhv_df[(bhv_df['Subject'].isin(train_list)) & (bhv_df['y'] == ii)]['Subject'].values print('No. of participants in class {} = {}'.format( ii, len(class_list[ii]))) ''' split participants in each class with kf nearly identical ratio of train and val, in all classes ''' split = {} for ii in range(args.k_class): split[ii] = kf.split(class_list[ii]) for i_fold in range(args.k_fold): _info('fold: %d/%d' % (i_fold + 1, args.k_fold)) # ***between-subject train-val split train_subs, val_subs = [], [] for ii in range(args.k_class): train, val = next(split[ii]) for jj in train: train_subs.append(class_list[ii][jj]) for jj in val: val_subs.append(class_list[ii][jj]) ''' model main ''' model = cpm(corr_thresh=args.corr_thresh) X_train, y_train, c_train = _get_seq(df, train_subs, args) X_val, y_val, c_val = _get_seq(df, val_subs, args) # train model _, _ = model.fit(X_train, y_train) ''' results on train data ''' s, s_c, _ = static_score(model, X_train, y_train, c_train, model_type=model_type) for score in SCORES: results['train_%s' % score][i_fold] = s[score] for ii in range(k_clip): results['c_train_%s' % score][ii][i_fold] = s_c[ii][score] print('train p = %0.3f' % s['p']) ''' results on val data ''' s, s_c, y_hat = static_score(model, X_val, y_val, c_val, model_type=model_type) for score in SCORES: results['val_%s' % score][i_fold] = s[score] for ii in range(k_clip): results['c_val_%s' % score][ii][i_fold] = s_c[ii][score] print('val p = %0.3f' % s['p']) results['y'][i_fold] = y_val results['y_hat'][i_fold] = y_hat results['c'][i_fold] = c_val return results
def _test(df, bhv_df, args): _info('test mode') # get X-y from df features = [ii for ii in df.columns if 'feat' in ii] k_feat = len(features) print('number of features = %d' % (k_feat)) k_clip = len(np.unique(df['c'])) print('number of clips = %d' % (k_clip)) subject_list = bhv_df['Subject'].unique() train_list = subject_list[:args.train_size] test_list = subject_list[args.train_size:] # init dict for all results results = {} for score in SCORES: # per clip score results['c_train_%s' % score] = {} results['c_test_%s' % score] = {} ''' model main ''' model = cpm(corr_thresh=args.corr_thresh) X_train, y_train, c_train = _get_seq(df, train_list, args) X_test, y_test, c_test = _get_seq(df, test_list, args) # train model _, _ = model.fit(X_train, y_train) ''' results on train data ''' s, s_c, _ = static_score(model, X_train, y_train, c_train, model_type=model_type) for score in SCORES: results['train_%s' % score] = s[score] for ii in range(k_clip): results['c_train_%s' % score][ii] = s_c[ii][score] print('train p = %0.3f' % s['p']) ''' results on test data ''' s, s_c, y_hat = static_score(model, X_test, y_test, c_test, model_type=model_type) for score in SCORES: results['test_%s' % score] = s[score] for ii in range(k_clip): results['c_test_%s' % score][ii] = s_c[ii][score] print('test p = %0.3f' % s['p']) results['y'] = y_test results['y_hat'] = y_hat results['c'] = c_test return results
def _train(df, bhv_df, args): # set pytorch device torch.manual_seed(K_SEED) use_cuda = torch.cuda.is_available() args.device = torch.device('cuda:0' if use_cuda else 'cpu') if use_cuda: _info('cuda') else: _info('cpu') # get X-y from df features = [ii for ii in df.columns if 'feat' in ii] k_feat = len(features) print('number of features = %d' % (k_feat)) k_clip = len(np.unique(df['c'])) print('number of clips = %d' % (k_clip)) # length of each clip clip_time = np.zeros(k_clip) for ii in range(k_clip): class_df = df[df['c'] == ii] clip_time[ii] = np.max(np.unique(class_df['timepoint'])) + 1 clip_time = clip_time.astype(int) # df saves float _info('seq lengths = %s' % clip_time) # init dict for all results results = {} # true and predicted scores and clip label results['y'] = {} results['y_hat'] = {} results['c'] = {} for score in SCORES: # mean scores across time results['train_%s' % score] = np.zeros(args.k_fold) results['val_%s' % score] = np.zeros(args.k_fold) # per clip temporal score results['t_train_%s' % score] = {} results['t_val_%s' % score] = {} for ii in range(k_clip): results['t_train_%s' % score][ii] = np.zeros( (args.k_fold, clip_time[ii])) results['t_val_%s' % score][ii] = np.zeros( (args.k_fold, clip_time[ii])) kf = KFold(n_splits=args.k_fold, random_state=K_SEED) # get participant lists for each assigned class class_list = {} for ii in range(args.k_class): class_list[ii] = bhv_df[bhv_df['y'] == ii]['Subject'].values ''' split participants in each class with kf nearly identical ratio of train and val, in all classes ''' split = {} for ii in range(args.k_class): split[ii] = kf.split(class_list[ii]) for i_fold in range(args.k_fold): _info('fold: %d/%d' % (i_fold + 1, args.k_fold)) # ***between-subject train-val split train_subs, val_subs = [], [] for ii in range(args.k_class): train, val = next(split[ii]) for jj in train: train_subs.append(class_list[ii][jj]) for jj in val: val_subs.append(class_list[ii][jj]) ''' model main ''' model = LSTMRegression(k_feat, args.k_hidden, args.k_layers) model.to(args.device) print(model) lossfn = nn.MSELoss() # if input is cuda, loss function is auto cuda opt = torch.optim.Adam(model.parameters()) # get train, val sequences X_train, train_len, y_train, c_train = _get_seq(df, train_subs, args) X_val, val_len, y_val, c_val = _get_seq(df, val_subs, args) max_length = torch.max(train_len) ''' train regression model ''' permutation = torch.randperm(X_train.size()[0]) losses = np.zeros(args.num_epochs) # then = time.time() for epoch in range(args.num_epochs): for i in range(0, X_train.size()[0], args.batch_size): indices = permutation[i:i + args.batch_size] batch_x, batch_y = X_train[indices], y_train[indices] batch_x_len = train_len[indices] batch_mask = torch.BoolTensor( _get_mask(batch_x_len, max_length)).to(args.device) y_pred = model(batch_x, batch_x_len, max_length).squeeze(2) loss = lossfn(y_pred[batch_mask == True], batch_y[batch_mask == True]) opt.zero_grad() loss.backward() opt.step() losses[epoch] = loss _info(losses) # print('--- train time = %0.4f seconds ---' % (time.time() - then)) model.eval() ''' results on train data ''' s, s_t, _, _, _ = _lstm_score(model, X_train, y_train, c_train, train_len, max_length, clip_time) for score in SCORES: results['train_%s' % score][i_fold] = s[score] for ii in range(k_clip): results['t_train_%s' % score][ii][i_fold] = s_t[ii][score] print('train p = %0.3f' % s['p']) ''' results on val data ''' s, s_t, y, y_hat, c = _lstm_score(model, X_val, y_val, c_val, val_len, max_length, clip_time) for score in SCORES: results['val_%s' % score][i_fold] = s[score] for ii in range(k_clip): results['t_val_%s' % score][ii][i_fold] = s_t[ii][score] print('val p = %0.3f' % s['p']) results['y'][i_fold] = y results['y_hat'][i_fold] = y_hat results['c'][i_fold] = c return results