def reduce_contrasts(
components: str = 'components_453_gm',
studies: Union[str, List[str]] = 'all',
masked_dir='unmasked',
output_dir='reduced',
n_jobs=1,
lstsq=False,
):
batch_size = 200
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if studies == 'all':
studies = STUDY_LIST
modl_atlas = fetch_atlas_modl()
mask = fetch_mask()
dictionary = modl_atlas[components]
masker = NiftiMasker(mask_img=mask).fit()
components = masker.transform(dictionary)
for study in studies:
this_data, targets = load(join(masked_dir, 'data_%s.pt' % study))
n_samples = this_data.shape[0]
batches = list(gen_batches(n_samples, batch_size))
this_data = Parallel(n_jobs=n_jobs,
verbose=10,
backend='multiprocessing',
mmap_mode='r')(delayed(single_reduce)(
components, this_data[batch], lstsq=lstsq)
for batch in batches)
this_data = np.concatenate(this_data, axis=0)
dump((this_data, targets), join(output_dir, 'data_%s.pt' % study))
def compute_classifs(estimator, standard_scaler, config, return_type='img'):
if config['model']['estimator'] in ['multi_study', 'ensemble']:
module = curate_module(estimator)
module.eval()
studies = module.classifiers.keys()
in_features = module.embedder.in_features
with torch.no_grad():
classifs = module(
{study: torch.eye(in_features)
for study in studies},
logits=True)
biases = module(
{study: torch.zeros((1, in_features))
for study in studies},
logits=True)
classifs = {
study: classifs[study] - biases[study]
for study in studies
}
classifs = {
study: classif - classif.mean(dim=0, keepdim=True)
for study, classif in classifs.items()
}
classifs = {
study: classif.numpy().T
for study, classif in classifs.items()
}
elif config['model']['estimator'] == 'logistic':
classifs = estimator.coef_
else:
raise ValueError('Wrong config file')
if standard_scaler is not None:
for study, classif in classifs.items():
sc = standard_scaler.scs_[study]
classifs[study] = classif / sc.scale_[None, :]
if config['data']['reduced']:
mask = fetch_mask()
masker = NiftiMasker(mask_img=mask).fit()
modl_atlas = fetch_atlas_modl()
dictionary = modl_atlas['components_453_gm']
dictionary = masker.transform(dictionary)
classifs = {
study: classif.dot(dictionary)
for study, classif in classifs.items()
}
if return_type == 'img':
classifs_img = masker.inverse_transform(
np.concatenate(list(classifs.values()), axis=0))
return classifs_img
elif return_type == 'arrays':
return classifs
else:
raise ValueError
def compute_rec():
mask_img = fetch_mask()
masker = MultiNiftiMasker(mask_img=mask_img).fit()
atlas = fetch_atlas_modl()
components_imgs = [
atlas.positive_new_components16, atlas.positive_new_components64,
atlas.positive_new_components512
]
components = masker.transform(components_imgs)
proj, proj_inv, rec = make_projection_matrix(components, scale_bases=True)
dump(rec, join(get_output_dir(), 'benchmark', 'rec.pkl'))
def test_atlas_modl():
"""Download the dataset in default directory and check success.
"""
atlas = fetch_atlas_modl()
keys = ['components_64',
'components_128',
'components_453_gm',
]
assert all(os.path.exists(atlas[key]) for key in keys)
def get_embedder_init(self):
weight = torch.empty_like(self.embedder.weight.data)
gain = 1. / math.sqrt(weight.shape[1])
if self.init == 'normal':
self.weight.data.uniform_(-gain, gain)
elif self.init == 'orthogonal':
nn.init.orthogonal_(weight, gain=gain)
elif self.init == 'resting-state':
dataset = fetch_atlas_modl()
weight = np.load(dataset['loadings_128_gm'])
weight = torch.from_numpy(np.array(weight))
return weight
def compute_components(estimator, config, return_type='img'):
"""Compute components from a FactoredClassifier estimator"""
module = curate_module(estimator)
components = module.embedder.linear.weight.detach().numpy()
if config['data']['reduced']:
mask = fetch_mask()
masker = NiftiMasker(mask_img=mask).fit()
modl_atlas = fetch_atlas_modl()
dictionary = modl_atlas['components_453_gm']
dictionary = masker.transform(dictionary)
components = components.dot(dictionary)
if return_type == 'img':
components_img = masker.inverse_transform(components)
return components_img
elif return_type == 'arrays':
return components
else:
raise ValueError
components = fetch_craddock_parcellation().parcellate400
data = np.ones_like(check_niimg(components).get_data())
mask = new_img_like(components, data)
label_masker = NiftiLabelsMasker(labels_img=components,
smoothing_fwhm=0,
mask_img=mask).fit()
maps_img = label_masker.inverse_transform(maps)
else:
mask = fetch_mask()
masker = MultiNiftiMasker(mask_img=mask).fit()
if source == 'msdl':
components = fetch_atlas_msdl()['maps']
components = masker.transform(components)
elif source in ['hcp_rs', 'hcp_rs_concat', 'hcp_rs_positive']:
data = fetch_atlas_modl()
if source == 'hcp_rs':
components_imgs = [data.nips2017_components64]
elif source == 'hcp_rs_concat':
components_imgs = [
data.nips2017_components16, data.nips2017_components64,
data.nips2017_components256
]
else:
components_imgs = [
data.positive_components16, data.positive_components64,
data.positive_components512
]
components = masker.transform(components_imgs)
proj, inv_proj, rec = memory.cache(make_projection_matrix)(
components, scale_bases=True)
def test_craddock_parcellation():
data = fetch_craddock_parcellation()
print(data)
data = fetch_atlas_modl()
print(data)
def reduce(dataset, output_dir=None, direct=False, source='hcp_rs_concat'):
"""Create a reduced version of a given dataset.
Unmask must be called beforehand"""
memory = Memory(cachedir=get_cache_dirs()[0], verbose=2)
print('Fetch data')
this_dataset_dir = join(get_output_dir(output_dir), 'unmasked', dataset)
masker, X = get_raw_contrast_data(this_dataset_dir)
print('Retrieve components')
if source == 'craddock':
components = fetch_craddock_parcellation().parcellate400
niimgs = masker.inverse_transform(X.values)
label_masker = NiftiLabelsMasker(labels_img=components,
smoothing_fwhm=0,
mask_img=masker.mask_img_).fit()
# components = label_masker.inverse_transform(np.eye(400))
print('Transform and fit data')
Xt = label_masker.transform(niimgs)
else:
if source == 'msdl':
components = [fetch_atlas_msdl()['maps']]
else:
data = fetch_atlas_modl()
if source == 'hcp_rs':
components_imgs = [data.nips2017_components256]
elif source == 'hcp_rs_concat':
components_imgs = [
data.nips2017_components16, data.nips2017_components64,
data.nips2017_components256
]
elif source == 'hcp_336':
components_imgs = [data.nips2017_components336]
elif source == 'hcp_new':
components_imgs = [
data.positive_new_components16,
data.positive_new_components64,
data.positive_new_components128
]
elif source == 'hcp_new_big':
components_imgs = [
data.positive_new_components16,
data.positive_new_components64,
data.positive_new_components512
]
elif source == 'hcp_rs_positive_concat':
components_imgs = [
data.positive_components16, data.positive_components64,
data.positive_components512
]
elif source == 'hcp_new_208':
components_imgs = [data.positive_new_components208]
components = masker.transform(components_imgs)
print('Transform and fit data')
proj, proj_inv, _ = memory.cache(make_projection_matrix)(
components, scale_bases=True)
if direct:
proj = proj_inv.T
Xt = X.dot(proj)
Xt = pd.DataFrame(data=Xt, index=X.index)
this_source = source
if direct:
this_source += '_direct'
this_output_dir = join(get_output_dir(output_dir), 'reduced', this_source,
dataset)
if not os.path.exists(this_output_dir):
os.makedirs(this_output_dir)
print(join(this_output_dir, 'Xt.pkl'))
Xt.to_pickle(join(this_output_dir, 'Xt.pkl'))
dump(masker, join(this_output_dir, 'masker.pkl'))
np.save(join(output_dir, 'components'), components)