def __init__(self, device, settings, data_type='aparc', test=False):
normalized_laplacian = True
self.coarsening_levels = 4
self.data_type = data_type
self.settings = settings
hcp_downloader = HCPDownloader(settings)
git_downloader = GitDownloader(settings)
self.loaders = [hcp_downloader, git_downloader]
#############
self.list_file = 'subjects.txt'
if test:
list_url = os.path.join(get_root(), 'conf/hcp/test/motor_lr',
self.list_file)
else:
list_url = os.path.join(get_root(), 'conf/hcp/train/motor_lr',
self.list_file)
#self.data_path = os.path.join(expanduser("~"), 'data_dense')
self.subjects = load_subjects(list_url)
self.p = 148
self.T = 284
self.session = 'MOTOR_LR'
self.transform = EncodePerm(15, 4, 4)
self.device = device
def load_hcp_vote(lookback=10):
list_file = 'subjects_inter.txt'
list_url = os.path.join(get_root(), 'conf', list_file)
subjects_strut = load_subjects(list_url)
list_file = 'subjects_hcp_all.txt'
# list_file = 'subjects_all.txt'
list_url = os.path.join(get_root(), 'conf', list_file)
subjects = load_subjects(list_url)
structural_file = 'struct_dti.mat'
structural_url = os.path.join(get_root(), 'load', 'hcpdata',
structural_file)
S = load_structural(subjects_strut, structural_url)
# data_path = '/Users/cassiano/Dropbox/cob/work/upenn/research/projects/tefemerid/code/v1/tfsid/out/data/hcp/many_motor'
# data_path = '~/data_hcp/'
data_path = os.path.join(os.path.expanduser("~"), 'data_full/aparc')
post_fix = '_aparc_tasks_aparc.mat'
# data_path = os.path.join(expanduser("~"), 'data_hcp')
# post_fix = '_aparc_tasks.mat'
p = 148
T = 284
C, X, _ = get_dataset(subjects,
data_path,
post_fix,
session='MOTOR_LR',
p=p,
T=T)
sh = C.shape
C, X = np.swapaxes(C, 1, 2), np.swapaxes(X, 1, 2)
C = C.reshape((C.shape[0] * C.shape[1], C.shape[2]))
X = X.reshape((X.shape[0] * X.shape[1], X.shape[2]))
assert (C.shape[0] == X.shape[0])
C = extend_signal(C)
# NONE is 1 - any(motor_task)
C[:, 0] = 1 - np.sum(C[:, 1:6], axis=1)
N_TRAIN = int(0.75 * X.shape[0])
X_train_1 = X[0:N_TRAIN, :]
labels_train_1 = C[0:N_TRAIN, :]
X_test_1 = X[N_TRAIN:, :]
labels_test_1 = C[N_TRAIN:, :]
X_train, y_train = get_lookback_data(X_train_1,
labels_train_1,
lookback=lookback)
X_test, y_test = get_lookback_data(X_test_1,
labels_test_1,
lookback=lookback)
X_train, X_test = np.swapaxes(X_train, 1, 2), np.swapaxes(X_test, 1, 2)
return X_train, y_train, X_test, y_test, S
def __init__(self):
normalized_laplacian = True
coarsening_levels = 4
list_file = 'subjects_inter.txt'
list_url = os.path.join(get_root(), 'conf', list_file)
subjects_strut = load_subjects(list_url)
structural_file = 'struct_dti.mat'
structural_url = os.path.join(get_root(), 'load', 'hcpdata',
structural_file)
S = load_strucutural(subjects_strut, structural_url)
S = S[0]
#avg_degree = 7
#S = scipy.sparse.random(65000, 65000, density=avg_degree/65000, format="csr")
self.graphs, self.perm = coarsening.coarsen(S,
levels=coarsening_levels,
self_connections=False)
self.list_file = 'subjects_hcp_all.txt'
list_url = os.path.join(get_root(), 'conf', self.list_file)
self.data_path = os.path.join(expanduser("~"), 'data_full')
self.subjects = load_subjects(list_url)
post_fix = '_aparc_tasks_aparc.mat'
self.filenames = [s + post_fix for s in self.subjects]
self.p = 148 #65000
self.T = 284
self.session = 'MOTOR_LR'
self.transform = EncodePerm(15, 4, 4, self.perm)
def load_hcp_example(full=False):
list_file = 'subjects_inter.txt'
list_url = os.path.join(get_root(), 'conf', list_file)
subjects_strut = load_subjects(list_url)
list_file = 'subjects_all.txt'
if full:
list_file = 'subjects_hcp_all.txt'
list_url = os.path.join(get_root(), 'conf', list_file)
subjects = load_subjects(list_url)
structural_file = 'struct_dti.mat'
structural_url = os.path.join(get_root(), 'load', 'hcpdata',
structural_file)
S = load_strucutural(subjects_strut, structural_url)
# data_path = '/Users/cassiano/Dropbox/cob/work/upenn/research/projects/tefemerid/code/v1/tfsid/out/data/hcp/many_motor'
# data_path = '~/data_hcp/'
data_path = os.path.join(expanduser("~"), 'data_hcp')
post_fix = '_aparc_tasks.mat'
if full:
data_path = os.path.join(expanduser("~"), 'data_full')
post_fix = '_aparc_tasks_aparc.mat'
p = 148
T = 284
C, X, _ = get_dataset(subjects,
data_path,
post_fix,
session='MOTOR_LR',
p=p,
T=T)
H, Gp, Gn = 15, 4, 4
Xw, y = encode(C, X, H, Gp, Gn)
N0 = np.nonzero(y == 0)[0].shape[0]
NN = int(np.nonzero(y > 0)[0].shape[0] / (np.unique(y).shape[0] - 1))
print('Ratio of class imbalance: {}'.format(N0 / NN))
ididx = np.random.permutation(np.nonzero(y == 0)[0].shape[0])[0:N0 - NN]
idx = np.nonzero(y == 0)[0][ididx]
# y = np.delete(y, idx, axis=0)
# Xw = np.delete(Xw, idx, axis=0)
one_hot = lambda x, k: np.array(x[:, None] == np.arange(k)[None, :],
dtype=int)
k = np.max(np.unique(y))
yoh = one_hot(y, k + 1)
return Xw, yoh, S
def __init__(self, settings, params):
self.logger = get_logger('HcpDataset')
self.local_folder = settings['DIRECTORIES']['local_server_directory']
self.parc = params['PARCELLATION']['parcellation']
self.inflation = params['SURFACE']['inflation']
self.tr = float(params['FMRI']['tr'])
self.physio_sampling_rate = int(params['FMRI']['physio_sampling_rate'])
self.regress_physio = params['FMRI']['regress_physio']
list_file = 'subjects_inter.txt'
list_url = os.path.join(get_root(), 'conf', list_file)
subjects_strut = load_subjects(list_url)
structural_file = 'struct_dti.mat'
structural_url = os.path.join(get_root(), 'conf', 'hcpdata',
structural_file)
self.S = load_strucutural(subjects_strut, structural_url)
def get_database_settings():
"""
Creates a ConfigParser object with server/directory/credentials/logging info from preconfigured directory.
:return: settings, a ConfigParser object
"""
settings = configparser.ConfigParser()
settings_furl = os.path.join(get_root(), 'dataset', 'hcp', 'conf', 'hcp_database.ini')
settings.read(settings_furl)
return settings
def load_hcp_example(list_file, session, horizon=15, Gp=4, Gn=4):
list_file_strut = 'subjects_inter.txt'
list_url_strut = os.path.join(get_root(), 'conf', list_file_strut)
subjects_strut = load_subjects(list_url_strut)
list_url = os.path.join(get_root(), 'conf', list_file)
subjects = load_subjects(list_url)
structural_file = 'struct_dti.mat'
structural_url = os.path.join(get_root(), 'load', 'hcpdata',
structural_file)
S = load_structural(subjects_strut, structural_url)
data_path = os.path.join(os.path.expanduser("~"), 'data_full')
post_fix = '_aparc_tasks_aparc.mat'
p = 148
T = 284
C, X, _ = get_dataset(subjects,
data_path,
post_fix,
session=session,
p=p,
T=T)
Xw, y = encode(C, X, horizon, Gp, Gn)
N0 = np.nonzero(y == 0)[0].shape[0]
NN = int(np.nonzero(y > 0)[0].shape[0] / (np.unique(y).shape[0] - 1))
print('Ratio of class imbalance: {:.3f}'.format(N0 / NN))
ididx = np.random.permutation(np.nonzero(y == 0)[0].shape[0])[0:N0 - NN]
idx = np.nonzero(y == 0)[0][ididx]
k = np.max(np.unique(y))
yoh = one_hot(y, k + 1)
return Xw, yoh, S
def __init__(self, perm):
self.list_file = 'subjects_test.txt'
list_url = os.path.join(get_root(), 'conf', self.list_file)
self.data_path = os.path.join(expanduser("~"), 'data_full')
self.subjects = load_subjects(list_url)
post_fix = '_aparc_tasks_aparc.mat'
self.filenames = [s + post_fix for s in self.subjects]
self.p = 148
self.T = 284
self.session = 'MOTOR_LR'
self.transform = Encode(15, 4, 4, perm)
def get_adjacency_dti(self, subject):
self.logger.debug("reading dti adjacency matrix for " + subject)
furl = os.path.join('HCP_1200', subject, 'MNINonLinear', 'Results',
'dMRI_CONN')
file = os.path.join(furl, subject + '.aparc.a2009s.dti.conn.mat')
token_url = os.path.join(furl, subject + '_404_token.txt')
self.dti_downloader.load(file, token_url)
# looks for local subject-specific DTI matrix
try:
dti_matrix = sio.loadmat(os.path.join(self.local_folder,
file)).get('S')
except FileNotFoundError:
# if not found, loads average DTI matrix from local resources directory
msg = "specific DTI matrix for " + subject + "," + self.parcellation + " not available, using average."
self.logger.info(msg)
relative_avg_furl = os.path.join(
'dataset', 'hcp', 'res', 'average1.aparc.a2009s.dti.conn.mat')
avg_furl = os.path.join(get_root(), relative_avg_furl)
try:
dti_matrix = sio.loadmat(avg_furl).get('S')
except:
msg = self.logger.error("average DTI file " + avg_furl +
" not found")
self.logger.error(msg)
raise SkipSubjectException(msg)
dti_coo = scipy.sparse.coo_matrix(dti_matrix)
self.logger.debug("done")
return dti_coo
def get_adj_dti(subject, parc, git_downloader):
#print("\n--- dti adjacency matrix...", end="", flush=True)
furl = os.path.join('HCP_1200', subject, 'MNINonLinear', 'Results',
'dMRI_CONN')
file = '/tmp/' + furl + '/' + subject + '.aparc.a2009s.dti.conn.mat'
git_downloader.load(file)
try:
S = sio.loadmat(file)
S = S.get('S')
except:
file_dir = os.path.join(get_root(),
'load/res/average1.aparc.a2009s.dti.conn.mat')
S = sio.loadmat(file_dir).get('S')
warnings.warn(
"Local DTI adjacency matrix for subject: <subject> in parcellation:<parcellation> not available, using average adjacency matrix."
)
S_coo = scipy.sparse.coo_matrix(S)
return S_coo
def main():
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size',
type=int,
default=1,
metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size',
type=int,
default=1000,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs',
type=int,
default=50,
metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--lr',
type=float,
default=0.001,
metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--momentum',
type=float,
default=0.5,
metavar='M',
help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model',
action='store_true',
default=False,
help='For Saving the current Model')
args = parser.parse_args()
args.reg_weight = 5.e-4
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
device = torch.device("cuda" if use_cuda else "cpu")
#device = torch.device("cpu")
normalized_laplacian = True
coarsening_levels = 4
settings = configparser.ConfigParser()
settings_dir = os.path.join(get_root(), 'load/res/hcp_loader.ini')
settings.read(settings_dir)
# kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
data_type = 'aparc'
if data_type == 'dense':
mat_size = 59412
else:
mat_size = 148
train_set = FullDataset(device, settings, data_type, test=False)
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=args.batch_size,
shuffle=False)
test_set = FullDataset(device, settings, data_type, test=True)
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=args.batch_size,
shuffle=False)
model = NetTGCNBasic(mat_size)
#model = NetMLP(int(graphs[0].shape[0] * 15))
if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
model.to(device)
pytorch_total_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print(pytorch_total_params)
#optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.95)
for epoch in range(1, args.epochs + 1):
t1 = time.time()
train_loss = train_minibatch(args,
model,
device,
train_loader,
optimizer,
epoch,
verbose=True)
scheduler.step()
test_loss, correct = test(args, model, device, test_loader, t1, epoch)
print(
'Epoch: {} Training loss: {:1.3e}, Test loss: {:1.3e}, Accuracy: {}/{} ({:.2f}%)'
.format(epoch, train_loss, test_loss, correct,
len(test_loader.dataset) * 270,
100. * correct / (len(test_loader.dataset) * 270)))
if args.save_model:
torch.save(model.state_dict(), "hcp_cnn_1gpu2.pt")
def read_surf_to_gray_map(hemi):
fname = os.path.join(get_root(), 'dataset', 'hcp', 'res',
hemi + '_dense_map.txt')
surf_to_gray = np.loadtxt(fname, delimiter=',', dtype=int)
return surf_to_gray
