def __init__(self, device, settings, data_type='aparc', test=False): normalized_laplacian = True self.coarsening_levels = 4 self.data_type = data_type self.settings = settings hcp_downloader = HCPDownloader(settings) git_downloader = GitDownloader(settings) self.loaders = [hcp_downloader, git_downloader] ############# self.list_file = 'subjects.txt' if test: list_url = os.path.join(get_root(), 'conf/hcp/test/motor_lr', self.list_file) else: list_url = os.path.join(get_root(), 'conf/hcp/train/motor_lr', self.list_file) #self.data_path = os.path.join(expanduser("~"), 'data_dense') self.subjects = load_subjects(list_url) self.p = 148 self.T = 284 self.session = 'MOTOR_LR' self.transform = EncodePerm(15, 4, 4) self.device = device
def load_hcp_vote(lookback=10): list_file = 'subjects_inter.txt' list_url = os.path.join(get_root(), 'conf', list_file) subjects_strut = load_subjects(list_url) list_file = 'subjects_hcp_all.txt' # list_file = 'subjects_all.txt' list_url = os.path.join(get_root(), 'conf', list_file) subjects = load_subjects(list_url) structural_file = 'struct_dti.mat' structural_url = os.path.join(get_root(), 'load', 'hcpdata', structural_file) S = load_structural(subjects_strut, structural_url) # data_path = '/Users/cassiano/Dropbox/cob/work/upenn/research/projects/tefemerid/code/v1/tfsid/out/data/hcp/many_motor' # data_path = '~/data_hcp/' data_path = os.path.join(os.path.expanduser("~"), 'data_full/aparc') post_fix = '_aparc_tasks_aparc.mat' # data_path = os.path.join(expanduser("~"), 'data_hcp') # post_fix = '_aparc_tasks.mat' p = 148 T = 284 C, X, _ = get_dataset(subjects, data_path, post_fix, session='MOTOR_LR', p=p, T=T) sh = C.shape C, X = np.swapaxes(C, 1, 2), np.swapaxes(X, 1, 2) C = C.reshape((C.shape[0] * C.shape[1], C.shape[2])) X = X.reshape((X.shape[0] * X.shape[1], X.shape[2])) assert (C.shape[0] == X.shape[0]) C = extend_signal(C) # NONE is 1 - any(motor_task) C[:, 0] = 1 - np.sum(C[:, 1:6], axis=1) N_TRAIN = int(0.75 * X.shape[0]) X_train_1 = X[0:N_TRAIN, :] labels_train_1 = C[0:N_TRAIN, :] X_test_1 = X[N_TRAIN:, :] labels_test_1 = C[N_TRAIN:, :] X_train, y_train = get_lookback_data(X_train_1, labels_train_1, lookback=lookback) X_test, y_test = get_lookback_data(X_test_1, labels_test_1, lookback=lookback) X_train, X_test = np.swapaxes(X_train, 1, 2), np.swapaxes(X_test, 1, 2) return X_train, y_train, X_test, y_test, S
def __init__(self): normalized_laplacian = True coarsening_levels = 4 list_file = 'subjects_inter.txt' list_url = os.path.join(get_root(), 'conf', list_file) subjects_strut = load_subjects(list_url) structural_file = 'struct_dti.mat' structural_url = os.path.join(get_root(), 'load', 'hcpdata', structural_file) S = load_strucutural(subjects_strut, structural_url) S = S[0] #avg_degree = 7 #S = scipy.sparse.random(65000, 65000, density=avg_degree/65000, format="csr") self.graphs, self.perm = coarsening.coarsen(S, levels=coarsening_levels, self_connections=False) self.list_file = 'subjects_hcp_all.txt' list_url = os.path.join(get_root(), 'conf', self.list_file) self.data_path = os.path.join(expanduser("~"), 'data_full') self.subjects = load_subjects(list_url) post_fix = '_aparc_tasks_aparc.mat' self.filenames = [s + post_fix for s in self.subjects] self.p = 148 #65000 self.T = 284 self.session = 'MOTOR_LR' self.transform = EncodePerm(15, 4, 4, self.perm)
def load_hcp_example(full=False): list_file = 'subjects_inter.txt' list_url = os.path.join(get_root(), 'conf', list_file) subjects_strut = load_subjects(list_url) list_file = 'subjects_all.txt' if full: list_file = 'subjects_hcp_all.txt' list_url = os.path.join(get_root(), 'conf', list_file) subjects = load_subjects(list_url) structural_file = 'struct_dti.mat' structural_url = os.path.join(get_root(), 'load', 'hcpdata', structural_file) S = load_strucutural(subjects_strut, structural_url) # data_path = '/Users/cassiano/Dropbox/cob/work/upenn/research/projects/tefemerid/code/v1/tfsid/out/data/hcp/many_motor' # data_path = '~/data_hcp/' data_path = os.path.join(expanduser("~"), 'data_hcp') post_fix = '_aparc_tasks.mat' if full: data_path = os.path.join(expanduser("~"), 'data_full') post_fix = '_aparc_tasks_aparc.mat' p = 148 T = 284 C, X, _ = get_dataset(subjects, data_path, post_fix, session='MOTOR_LR', p=p, T=T) H, Gp, Gn = 15, 4, 4 Xw, y = encode(C, X, H, Gp, Gn) N0 = np.nonzero(y == 0)[0].shape[0] NN = int(np.nonzero(y > 0)[0].shape[0] / (np.unique(y).shape[0] - 1)) print('Ratio of class imbalance: {}'.format(N0 / NN)) ididx = np.random.permutation(np.nonzero(y == 0)[0].shape[0])[0:N0 - NN] idx = np.nonzero(y == 0)[0][ididx] # y = np.delete(y, idx, axis=0) # Xw = np.delete(Xw, idx, axis=0) one_hot = lambda x, k: np.array(x[:, None] == np.arange(k)[None, :], dtype=int) k = np.max(np.unique(y)) yoh = one_hot(y, k + 1) return Xw, yoh, S
def __init__(self, settings, params): self.logger = get_logger('HcpDataset') self.local_folder = settings['DIRECTORIES']['local_server_directory'] self.parc = params['PARCELLATION']['parcellation'] self.inflation = params['SURFACE']['inflation'] self.tr = float(params['FMRI']['tr']) self.physio_sampling_rate = int(params['FMRI']['physio_sampling_rate']) self.regress_physio = params['FMRI']['regress_physio'] list_file = 'subjects_inter.txt' list_url = os.path.join(get_root(), 'conf', list_file) subjects_strut = load_subjects(list_url) structural_file = 'struct_dti.mat' structural_url = os.path.join(get_root(), 'conf', 'hcpdata', structural_file) self.S = load_strucutural(subjects_strut, structural_url)
def get_database_settings(): """ Creates a ConfigParser object with server/directory/credentials/logging info from preconfigured directory. :return: settings, a ConfigParser object """ settings = configparser.ConfigParser() settings_furl = os.path.join(get_root(), 'dataset', 'hcp', 'conf', 'hcp_database.ini') settings.read(settings_furl) return settings
def load_hcp_example(list_file, session, horizon=15, Gp=4, Gn=4): list_file_strut = 'subjects_inter.txt' list_url_strut = os.path.join(get_root(), 'conf', list_file_strut) subjects_strut = load_subjects(list_url_strut) list_url = os.path.join(get_root(), 'conf', list_file) subjects = load_subjects(list_url) structural_file = 'struct_dti.mat' structural_url = os.path.join(get_root(), 'load', 'hcpdata', structural_file) S = load_structural(subjects_strut, structural_url) data_path = os.path.join(os.path.expanduser("~"), 'data_full') post_fix = '_aparc_tasks_aparc.mat' p = 148 T = 284 C, X, _ = get_dataset(subjects, data_path, post_fix, session=session, p=p, T=T) Xw, y = encode(C, X, horizon, Gp, Gn) N0 = np.nonzero(y == 0)[0].shape[0] NN = int(np.nonzero(y > 0)[0].shape[0] / (np.unique(y).shape[0] - 1)) print('Ratio of class imbalance: {:.3f}'.format(N0 / NN)) ididx = np.random.permutation(np.nonzero(y == 0)[0].shape[0])[0:N0 - NN] idx = np.nonzero(y == 0)[0][ididx] k = np.max(np.unique(y)) yoh = one_hot(y, k + 1) return Xw, yoh, S
def __init__(self, perm): self.list_file = 'subjects_test.txt' list_url = os.path.join(get_root(), 'conf', self.list_file) self.data_path = os.path.join(expanduser("~"), 'data_full') self.subjects = load_subjects(list_url) post_fix = '_aparc_tasks_aparc.mat' self.filenames = [s + post_fix for s in self.subjects] self.p = 148 self.T = 284 self.session = 'MOTOR_LR' self.transform = Encode(15, 4, 4, perm)
def get_adjacency_dti(self, subject): self.logger.debug("reading dti adjacency matrix for " + subject) furl = os.path.join('HCP_1200', subject, 'MNINonLinear', 'Results', 'dMRI_CONN') file = os.path.join(furl, subject + '.aparc.a2009s.dti.conn.mat') token_url = os.path.join(furl, subject + '_404_token.txt') self.dti_downloader.load(file, token_url) # looks for local subject-specific DTI matrix try: dti_matrix = sio.loadmat(os.path.join(self.local_folder, file)).get('S') except FileNotFoundError: # if not found, loads average DTI matrix from local resources directory msg = "specific DTI matrix for " + subject + "," + self.parcellation + " not available, using average." self.logger.info(msg) relative_avg_furl = os.path.join( 'dataset', 'hcp', 'res', 'average1.aparc.a2009s.dti.conn.mat') avg_furl = os.path.join(get_root(), relative_avg_furl) try: dti_matrix = sio.loadmat(avg_furl).get('S') except: msg = self.logger.error("average DTI file " + avg_furl + " not found") self.logger.error(msg) raise SkipSubjectException(msg) dti_coo = scipy.sparse.coo_matrix(dti_matrix) self.logger.debug("done") return dti_coo
def get_adj_dti(subject, parc, git_downloader): #print("\n--- dti adjacency matrix...", end="", flush=True) furl = os.path.join('HCP_1200', subject, 'MNINonLinear', 'Results', 'dMRI_CONN') file = '/tmp/' + furl + '/' + subject + '.aparc.a2009s.dti.conn.mat' git_downloader.load(file) try: S = sio.loadmat(file) S = S.get('S') except: file_dir = os.path.join(get_root(), 'load/res/average1.aparc.a2009s.dti.conn.mat') S = sio.loadmat(file_dir).get('S') warnings.warn( "Local DTI adjacency matrix for subject: <subject> in parcellation:<parcellation> not available, using average adjacency matrix." ) S_coo = scipy.sparse.coo_matrix(S) return S_coo
def main(): # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--batch-size', type=int, default=1, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=50, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=False, help='For Saving the current Model') args = parser.parse_args() args.reg_weight = 5.e-4 use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) device = torch.device("cuda" if use_cuda else "cpu") #device = torch.device("cpu") normalized_laplacian = True coarsening_levels = 4 settings = configparser.ConfigParser() settings_dir = os.path.join(get_root(), 'load/res/hcp_loader.ini') settings.read(settings_dir) # kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} data_type = 'aparc' if data_type == 'dense': mat_size = 59412 else: mat_size = 148 train_set = FullDataset(device, settings, data_type, test=False) train_loader = torch.utils.data.DataLoader(train_set, batch_size=args.batch_size, shuffle=False) test_set = FullDataset(device, settings, data_type, test=True) test_loader = torch.utils.data.DataLoader(test_set, batch_size=args.batch_size, shuffle=False) model = NetTGCNBasic(mat_size) #model = NetMLP(int(graphs[0].shape[0] * 15)) if torch.cuda.device_count() > 1: model = torch.nn.DataParallel(model) model.to(device) pytorch_total_params = sum(p.numel() for p in model.parameters() if p.requires_grad) print(pytorch_total_params) #optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) optimizer = optim.Adam(model.parameters(), lr=args.lr) scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.95) for epoch in range(1, args.epochs + 1): t1 = time.time() train_loss = train_minibatch(args, model, device, train_loader, optimizer, epoch, verbose=True) scheduler.step() test_loss, correct = test(args, model, device, test_loader, t1, epoch) print( 'Epoch: {} Training loss: {:1.3e}, Test loss: {:1.3e}, Accuracy: {}/{} ({:.2f}%)' .format(epoch, train_loss, test_loss, correct, len(test_loader.dataset) * 270, 100. * correct / (len(test_loader.dataset) * 270))) if args.save_model: torch.save(model.state_dict(), "hcp_cnn_1gpu2.pt")
def read_surf_to_gray_map(hemi): fname = os.path.join(get_root(), 'dataset', 'hcp', 'res', hemi + '_dense_map.txt') surf_to_gray = np.loadtxt(fname, delimiter=',', dtype=int) return surf_to_gray