def remote_dir_is_writable(user, hosts, path):
"""
Test if *path* is writable from each host in *hosts*. Sending bash
commands to each host via ssh using the given *user* login.
Args:
"""
import os.path as op
import pyhrf
mode = 'dispatch'
cmds = ['bash -c "echo -n \"[write_test]:%d:\"; '\
'if [ -w %s ]; then echo \"OK\"; else echo \"NOTOK\"; fi;"' \
%(ih, path) for ih in range(len(hosts))]
tasks = read_tasks(cmds, mode)
timeslot = read_timeslot('allday')
tmp_dir = pyhrf.get_tmp_path()
logfile = op.join(tmp_dir, 'pyhrf.log')
run_grid(mode, hosts, 'rsa', tasks, timeslot, logfile=logfile, user=user)
kill_threads()
log = open(logfile).readlines()
res = [False] * len(hosts)
for line in log:
if line.startswith('[write_test]'):
#print line
_, ih, r = line.strip('\n').split(':')
res[int(ih)] = ('OK' in r)
os.remove(logfile)
return res
def setUp(self):
pyhrf.verbose.set_verbosity(0)
np.random.seed(8652761)
self.tmp_dir = pyhrf.get_tmp_path()
def _test_spm_option_parse(self, spm_ver):
"""
Test parsing of option "-s SPM.mat" with given SPM version (int)
"""
spm_file = op.join(pyhrf.get_tmp_path(), 'SPM.mat')
tools._io._zip.gunzip(pyhrf.get_data_file_name('SPM_v%d.mat.gz' % spm_ver),
outFileName=spm_file)
options = ['-s', spm_file]
from optparse import OptionParser
parser = OptionParser()
ptr.append_common_treatment_options(parser)
fd = ptr.parse_data_options(parser.parse_args(options)[0])
self.assertEqual(fd.tr, 2.4) # nb sessions
p = fd.paradigm
# nb sessions
self.assertEqual(len(p.stimOnsets[p.stimOnsets.keys()[0]]), 2)
npt.assert_almost_equal(p.stimOnsets['audio'][0],
ppar.onsets_loc_av['audio'][0])
npt.assert_almost_equal(p.stimOnsets['audio'][1],
ppar.onsets_loc_av['audio'][0])
npt.assert_almost_equal(p.stimOnsets['video'][1],
ppar.onsets_loc_av['video'][0])
def remote_dir_is_writable(user, hosts, path):
"""
Test if *path* is writable from each host in *hosts*. Sending bash
commands to each host via ssh using the given *user* login.
Args:
"""
import os.path as op
import pyhrf
mode = 'dispatch'
cmds = ['bash -c "echo -n \"[write_test]:%d:\"; '\
'if [ -w %s ]; then echo \"OK\"; else echo \"NOTOK\"; fi;"' \
%(ih, path) for ih in range(len(hosts))]
tasks = read_tasks(cmds, mode)
timeslot = read_timeslot('allday')
tmp_dir = pyhrf.get_tmp_path()
brokenfile = op.join(tmp_dir, 'pyhrf-broken_cmd.batch')
logfile = op.join(tmp_dir, 'pyhrf-parallel.log')
run_grid(mode, hosts, 'rsa', tasks, timeslot, brokenfile,
logfile, user=user)
kill_threads()
log = open(logfile).readlines()
res = [False] * len(hosts)
for line in log:
if line.startswith('[write_test]'):
#print line
_,ih,r = line.strip('\n').split(':')
res[int(ih)] = ('OK' in r)
return res
def setUp(self):
np.random.seed(8652761)
self.tmp_dir = pyhrf.get_tmp_path()
self.clean_tmp = True
self.sampler_params_for_single_test = {
'nb_iterations':
40,
'smpl_hist_pace':
1,
'obs_hist_pace':
1,
'brf':
jasl.PhysioBOLDResponseSampler(do_sampling=False,
normalise=1.,
use_true_value=True,
zero_constraint=False),
'brf_var':
jasl.PhysioBOLDResponseVarianceSampler(do_sampling=False,
val_ini=np.array([0.1]),
use_true_value=False),
'prf':
jasl.PhysioPerfResponseSampler(
do_sampling=False,
normalise=1.,
use_true_value=True,
zero_constraint=False,
prior_type='physio_stochastic_regularized'),
'prf_var':
jasl.PhysioPerfResponseVarianceSampler(do_sampling=False,
val_ini=np.array([.001]),
use_true_value=False),
'noise_var':
jasl.NoiseVarianceSampler(do_sampling=False, use_true_value=True),
'drift_var':
jasl.DriftVarianceSampler(do_sampling=False, use_true_value=True),
'drift':
jasl.DriftCoeffSampler(do_sampling=False, use_true_value=True),
'bold_response_levels':
jasl.BOLDResponseLevelSampler(do_sampling=False,
use_true_value=True),
'perf_response_levels':
jasl.PerfResponseLevelSampler(do_sampling=False,
use_true_value=True),
'labels':
jasl.LabelSampler(do_sampling=False, use_true_value=True),
'bold_mixt_params':
jasl.BOLDMixtureSampler(do_sampling=False, use_true_value=True),
'perf_mixt_params':
jasl.PerfMixtureSampler(do_sampling=False, use_true_value=True),
'perf_baseline':
jasl.PerfBaselineSampler(do_sampling=False, use_true_value=True),
'perf_baseline_var':
jasl.PerfBaselineVarianceSampler(do_sampling=False,
use_true_value=True),
'check_final_value':
'raise', # print or raise
}
def setUp(self):
self.tmp_dir = pyhrf.get_tmp_path()
simu = simulate_sessions(output_dir=self.tmp_dir,
snr_scenario='high_snr',
spatial_size='random_small')
self.data_simu = merge_fmri_sessions(simu)
def setUp(self):
cache_dir = tempfile.mkdtemp(prefix='pyhrf_validate',
dir=pyhrf.cfg['global']['tmp_path'])
mem = Memory(cache_dir)
np.random.seed(8652761)
self.tmp_dir = pyhrf.get_tmp_path()
self.clean_tmp = True
def setUp(self):
# pyhrf.verbose.set_verbosity(0)
pyhrf.logger.setLevel(logging.WARNING)
np.random.seed(8652761)
self.tmp_dir = pyhrf.get_tmp_path()
def _test_load_regnames(self, spm_ver):
spm_file = op.join(pyhrf.get_tmp_path(), 'SPM.mat')
pio._zip.gunzip(pyhrf.get_data_file_name('SPM_v%d.mat.gz' % spm_ver),
outFileName=spm_file)
expected = ['Sn(1) audio*bf(1)', 'Sn(1) video*bf(1)',
'Sn(2) audio*bf(1)', 'Sn(2) video*bf(1)',
'Sn(1) constant', 'Sn(2) constant']
self.assertEqual(pio.spmio.load_regnames(spm_file), expected)
def setUp(self):
np.random.seed(8652761)
self.simu_dir = pyhrf.get_tmp_path()
# Parameters to setup a Sampler where all samplers are OFF and
# set to their true values.
# This is used by the function _test_specific_samplers,
# which will turn on specific samplers to test.
self.sampler_params_for_single_test = {
'nb_iterations': 40,
'smpl_hist_pace': -1,
'obs_hist_pace': -1,
# HRF by subject
'hrf_subj': jms.HRF_Sampler(do_sampling=False,
normalise=1.,
use_true_value=True,
zero_contraint=False,
prior_type='singleHRF'),
# HRF variance
'hrf_var_subj': jms.HRFVarianceSubjectSampler(do_sampling=False,
use_true_value=True),
# HRF group
'hrf_group': jms.HRF_Group_Sampler(do_sampling=False,
normalise=1.,
use_true_value=True,
zero_contraint=False,
prior_type='singleHRF'),
# HRF variance
'hrf_var_group': jms.RHGroupSampler(do_sampling=False,
use_true_value=True),
# neural response levels (stimulus-induced effects) by subject
'response_levels': jms.NRLs_Sampler(do_sampling=False,
use_true_value=True),
'labels': jms.LabelSampler(do_sampling=False,
use_true_value=True),
# drift
'drift': jms.Drift_MultiSubj_Sampler(do_sampling=False,
use_true_value=True),
# drift variance
'drift_var': jms.ETASampler_MultiSubj(do_sampling=False,
use_true_value=True),
# noise variance
'noise_var':
jms.NoiseVariance_Drift_MultiSubj_Sampler(do_sampling=False,
use_true_value=False),
# weights o fthe mixture
# parameters of the mixture
'mixt_params': jms.MixtureParamsSampler(do_sampling=False,
use_true_value=False),
#'alpha_subj' : Alpha_hgroup_Sampler(dict_alpha_single),
#'alpha_var_subj' : AlphaVar_Sampler(dict_alpha_var_single),
'check_final_value': 'none', # print or raise
}
def setUp(self):
tag = "subj0_%s.nii.gz"
self.func_file = pyhrf.get_data_file_name(tag % "bold_session0")
self.anatomy_file = pyhrf.get_data_file_name(tag % "anatomy")
self.roi_mask_file = pyhrf.get_data_file_name(tag % "parcellation")
self.ax_slice = 24
self.sag_slice = 7
self.cor_slice = 34
self.tmp_dir = pyhrf.get_tmp_path() #'./'
def setUp(self):
tag = 'subj0_%s.nii.gz'
self.func_file = pyhrf.get_data_file_name(tag%'bold_session0')
self.anatomy_file = pyhrf.get_data_file_name(tag%'anatomy')
self.roi_mask_file = pyhrf.get_data_file_name(tag%'parcellation')
self.ax_slice = 24
self.sag_slice = 7
self.cor_slice = 34
self.tmp_dir = pyhrf.get_tmp_path() #'./'
def setUp(self):
tag = 'subj0_%s.nii.gz'
self.func_file = pyhrf.get_data_file_name(tag % 'bold_session0')
self.anatomy_file = pyhrf.get_data_file_name(tag % 'anatomy')
self.roi_mask_file = pyhrf.get_data_file_name(tag % 'parcellation')
self.ax_slice = 24
self.sag_slice = 7
self.cor_slice = 34
self.tmp_dir = pyhrf.get_tmp_path()
def setUp(self):
np.random.seed(8652761)
self.tmp_dir = pyhrf.get_tmp_path()
self.clean_tmp = True
self.sampler_params_for_single_test = {
'nb_iterations': 40,
'smpl_hist_pace': 1,
'obs_hist_pace': 1,
'brf': jasl.PhysioBOLDResponseSampler(do_sampling=False,
normalise=1.,
use_true_value=True,
zero_constraint=False),
'brf_var':
jasl.PhysioBOLDResponseVarianceSampler(do_sampling=False,
val_ini=np.array([0.1]),
use_true_value=False),
'prf': jasl.PhysioPerfResponseSampler(do_sampling=False,
normalise=1.,
use_true_value=True,
zero_constraint=False,
prior_type='physio_stochastic_regularized'),
'prf_var':
jasl.PhysioPerfResponseVarianceSampler(do_sampling=False,
val_ini=np.array(
[.001]),
use_true_value=False),
'noise_var': jasl.NoiseVarianceSampler(do_sampling=False,
use_true_value=True),
'drift_var': jasl.DriftVarianceSampler(do_sampling=False,
use_true_value=True),
'drift': jasl.DriftCoeffSampler(do_sampling=False,
use_true_value=True),
'bold_response_levels':
jasl.BOLDResponseLevelSampler(do_sampling=False,
use_true_value=True),
'perf_response_levels':
jasl.PerfResponseLevelSampler(do_sampling=False,
use_true_value=True),
'labels': jasl.LabelSampler(do_sampling=False,
use_true_value=True),
'bold_mixt_params': jasl.BOLDMixtureSampler(do_sampling=False,
use_true_value=True),
'perf_mixt_params': jasl.PerfMixtureSampler(do_sampling=False,
use_true_value=True),
'perf_baseline': jasl.PerfBaselineSampler(do_sampling=False,
use_true_value=True),
'perf_baseline_var':
jasl.PerfBaselineVarianceSampler(do_sampling=False,
use_true_value=True),
'check_final_value': 'raise', # print or raise
}
def dump_roi_datasets(self, dry=False, output_dir=None):
pyhrf.verbose(1,'Loading data ...')
# if no file to dump (dry), assume it's only to get file names,
# then don't build the graph (could take some time ...)
if not dry:
self.data.build_graphs()
explData = self.analyser.split_data(self.data)
files = []
roiIds = []
if output_dir is not None:
roi_data_out_dir = output_dir
else:
if self.output_dir is not None:
roi_data_out_dir = op.join(self.output_dir, 'ROI_datasets')
else:
roi_data_out_dir = op.join(pyhrf.get_tmp_path(), 'ROI_datasets')
if not op.exists(roi_data_out_dir): os.makedirs(roi_data_out_dir)
assert op.exists(roi_data_out_dir)
if not dry:
pyhrf.verbose(1,'Dump roi data in dir %s' %roi_data_out_dir)
#data_order = sorted([d.get_nb_vox_in_mask() for d in explData])
pyhrf.verbose(1,'Dump of roi data, ordering by size ...')
cmp_size = lambda e1,e2:cmp(e1.get_nb_vox_in_mask(),
e2.get_nb_vox_in_mask())
for edata in sorted(explData, cmp=cmp_size, reverse=True):
roiId = edata.get_roi_id()
fn = op.abspath(op.join(roi_data_out_dir,
"roidata_%04d.pck" %roiId))
roiIds.append(roiId)
if not dry:
f = open(fn ,'w')
cPickle.dump(edata, f)
f.close()
files.append(fn)
pyhrf.verbose(1,'Dump of roi data done.')
return files, roiIds
def dump_roi_datasets(self, dry=False, output_dir=None):
logger.info('Loading data ...')
# if no file to dump (dry), assume it's only to get file names,
# then don't build the graph (could take some time ...)
if not dry:
self.data.build_graphs()
explData = self.analyser.split_data(self.data)
files = []
roiIds = []
if output_dir is not None:
roi_data_out_dir = output_dir
else:
if self.output_dir is not None:
roi_data_out_dir = op.join(self.output_dir, 'ROI_datasets')
else:
roi_data_out_dir = op.join(pyhrf.get_tmp_path(),
'ROI_datasets')
if not op.exists(roi_data_out_dir):
os.makedirs(roi_data_out_dir)
assert op.exists(roi_data_out_dir)
if not dry:
logger.info('Dump roi data in dir %s', roi_data_out_dir)
logger.info('Dump of roi data, ordering by size ...')
cmp_size = lambda e1, e2: cmp(e1.get_nb_vox_in_mask(),
e2.get_nb_vox_in_mask())
for edata in sorted(explData, cmp=cmp_size, reverse=True):
roiId = edata.get_roi_id()
fn = op.abspath(
op.join(roi_data_out_dir, "roidata_%04d.pck" % roiId))
roiIds.append(roiId)
if not dry:
f = open(fn, 'w')
cPickle.dump(edata, f)
f.close()
files.append(fn)
logger.info('Dump of roi data done.')
return files, roiIds
def setUp(self):
from pyhrf.ndarray import MRI3Daxes
self.tmp_dir = pyhrf.get_tmp_path()
self.p1 = np.array([[[1, 1, 1, 3],
[1, 1, 3, 3],
[0, 1, 2, 2],
[0, 2, 2, 2],
[0, 0, 2, 4]]], dtype=np.int32)
self.p1_fn = op.join(self.tmp_dir, 'p1.nii')
xndarray(self.p1, axes_names=MRI3Daxes).save(self.p1_fn)
self.p2 = self.p1 * 4.5
self.p2_fn = op.join(self.tmp_dir, 'p2.nii')
xndarray(self.p2, axes_names=MRI3Daxes).save(self.p2_fn)
self.mask = (self.p1 > 0).astype(np.int32)
self.mask_fn = op.join(self.tmp_dir, 'mask.nii')
xndarray(self.mask, axes_names=MRI3Daxes).save(self.mask_fn)
def make_parcellation_cubed_blobs_from_file(parcellation_file,
output_path,
roi_ids=None,
bg_parcel=0,
skip_existing=False):
p, mp = read_volume(parcellation_file)
p = p.astype(np.int32)
if bg_parcel == 0 and p.min() == -1:
p += 1 # set background to 0
if roi_ids is None:
roi_ids = np.unique(p)
logger.info('%d rois to extract', (len(roi_ids) - 1))
tmp_dir = pyhrf.get_tmp_path('blob_parcellation')
tmp_parcel_mask_file = op.join(tmp_dir, 'parcel_for_blob.nii')
out_files = []
for roi_id in roi_ids:
if roi_id != bg_parcel: # discard background
output_blob_file = op.join(output_path,
'parcel_%d_cubed_blob.arg' % roi_id)
out_files.append(output_blob_file)
if skip_existing and os.path.exists(output_blob_file):
continue
parcel_mask = (p == roi_id).astype(np.int32)
write_volume(parcel_mask, tmp_parcel_mask_file, mp)
logger.info('Extract ROI %d -> %s', roi_id, output_blob_file)
cmd = 'AimsGraphConvert -i %s -o %s --bucket' \
% (tmp_parcel_mask_file, output_blob_file)
logger.info('Cmd: %s', cmd)
os.system(cmd)
if op.exists(tmp_parcel_mask_file):
os.remove(tmp_parcel_mask_file)
return out_files
def make_parcellation_cubed_blobs_from_file(parcellation_file, output_path,
roi_ids=None, bg_parcel=0,
skip_existing=False):
p,mp = read_volume(parcellation_file)
p = p.astype(np.int32)
if bg_parcel==0 and p.min() == -1:
p += 1 #set background to 0
if roi_ids is None:
roi_ids = np.unique(p)
pyhrf.verbose(1,'%d rois to extract' %(len(roi_ids)-1))
tmp_dir = pyhrf.get_tmp_path('blob_parcellation')
tmp_parcel_mask_file = op.join(tmp_dir, 'parcel_for_blob.nii')
out_files = []
for roi_id in roi_ids:
if roi_id != bg_parcel: #discard background
output_blob_file = op.join(output_path, 'parcel_%d_cubed_blob.arg'\
%roi_id)
out_files.append(output_blob_file)
if skip_existing and os.path.exists(output_blob_file):
continue
parcel_mask = (p==roi_id).astype(np.int32)
write_volume(parcel_mask, tmp_parcel_mask_file, mp)
pyhrf.verbose(3,'Extract ROI %d -> %s' %(roi_id,output_blob_file))
cmd = 'AimsGraphConvert -i %s -o %s --bucket' \
%(tmp_parcel_mask_file, output_blob_file)
pyhrf.verbose(3,'Cmd: %s' %(cmd))
os.system(cmd)
if op.exists(tmp_parcel_mask_file):
os.remove(tmp_parcel_mask_file)
return out_files
def setUp(self):
self.tmp_dir = pyhrf.get_tmp_path()
simu = simulate_sessions(output_dir=self.tmp_dir,
snr_scenario='high_snr',
spatial_size='random_small')
self.data_simu = merge_fmri_sessions(simu)
def setUp(self):
np.random.seed(8652761)
self.tmp_dir = pyhrf.get_tmp_path()
def setUp(self):
np.random.seed(8652761)
self.simu_dir = pyhrf.get_tmp_path()
# Parameters to setup a Sampler where all samplers are OFF and
# set to their true values.
# This is used by the function _test_specific_samplers,
# which will turn on specific samplers to test.
self.sampler_params_for_single_test = {
'nb_iterations':
40,
'smpl_hist_pace':
-1,
'obs_hist_pace':
-1,
# HRF by subject
'hrf_subj':
jms.HRF_Sampler(do_sampling=False,
normalise=1.,
use_true_value=True,
zero_contraint=False,
prior_type='singleHRF'),
# HRF variance
'hrf_var_subj':
jms.HRFVarianceSubjectSampler(do_sampling=False,
use_true_value=True),
# HRF group
'hrf_group':
jms.HRF_Group_Sampler(do_sampling=False,
normalise=1.,
use_true_value=True,
zero_contraint=False,
prior_type='singleHRF'),
# HRF variance
'hrf_var_group':
jms.RHGroupSampler(do_sampling=False, use_true_value=True),
# neural response levels (stimulus-induced effects) by subject
'response_levels':
jms.NRLs_Sampler(do_sampling=False, use_true_value=True),
'labels':
jms.LabelSampler(do_sampling=False, use_true_value=True),
# drift
'drift':
jms.Drift_MultiSubj_Sampler(do_sampling=False,
use_true_value=True),
# drift variance
'drift_var':
jms.ETASampler_MultiSubj(do_sampling=False, use_true_value=True),
# noise variance
'noise_var':
jms.NoiseVariance_Drift_MultiSubj_Sampler(do_sampling=False,
use_true_value=False),
# weights o fthe mixture
# parameters of the mixture
'mixt_params':
jms.MixtureParamsSampler(do_sampling=False, use_true_value=False),
#'alpha_subj' : Alpha_hgroup_Sampler(dict_alpha_single),
#'alpha_var_subj' : AlphaVar_Sampler(dict_alpha_var_single),
'check_final_value':
'none', # print or raise
}
def project_fmri_from_kernels(input_mesh, kernels_file, fmri_data_file, output_tex, bin_threshold=None):
logger.info("Project data onto mesh using kernels ...")
if 0:
print "Projecting ..."
print "func data:", fmri_data_file
print "Mesh file:", input_mesh
print "Save as:", output_tex
logger.info("Call AimsFunctionProjection -op 1 ...")
data_files = []
output_texs = []
p_ids = None
if bin_threshold is not None:
d, h = read_volume(fmri_data_file)
if np.allclose(d.astype(int), d):
tmp_dir = pyhrf.get_tmp_path()
p_ids = np.unique(d)
logger.info("bin threshold: %f", bin_threshold)
logger.info("pids(n=%d): %d...%d", len(p_ids), min(p_ids), max(p_ids))
for i, p_id in enumerate(p_ids):
if p_id != 0:
new_p = np.zeros_like(d)
new_p[np.where(d == p_id)] = i + 1 # 0 is background
ifn = op.join(tmp_dir, "pmask_%d.nii" % p_id)
write_volume(new_p, ifn, h)
data_files.append(ifn)
ofn = op.join(tmp_dir, "ptex_%d.gii" % p_id)
output_texs.append(ofn)
else:
data_files.append(fmri_data_file)
output_texs.append(output_tex)
else:
data_files.append(fmri_data_file)
output_texs.append(output_tex)
logger.info("input data files: %s", str(data_files))
logger.info("output data files: %s", str(output_texs))
for data_file, o_tex in zip(data_files, output_texs):
projection = [
"AimsFunctionProjection",
"-op",
"1",
"-d",
kernels_file,
"-d1",
data_file,
"-m",
input_mesh,
"-o",
o_tex,
]
cmd = " ".join(map(str, projection))
logger.info("cmd: %s", cmd)
os.system(cmd)
if bin_threshold is not None:
logger.info("Binary threshold of texture at %f", bin_threshold)
o_tex = output_texs[0]
data, data_gii = read_texture(o_tex)
data = (data > bin_threshold).astype(np.int32)
print "data:", data.dtype
if p_ids is not None:
for pid, o_tex in zip(p_ids[1:], output_texs[1:]):
pdata, pdata_gii = read_texture(o_tex)
data += (pdata > bin_threshold).astype(np.int32) * pid
# assert (np.unique(data) == p_ids).all()
write_texture(data, output_tex, intent="NIFTI_INTENT_LABEL")
def project_fmri_from_kernels(input_mesh, kernels_file, fmri_data_file,
output_tex, bin_threshold=None, ):
pyhrf.verbose(2,'Project data onto mesh using kernels ...')
if 0:
print 'Projecting ...'
print 'func data:', fmri_data_file
print 'Mesh file:', input_mesh
print 'Save as:', output_tex
pyhrf.verbose(2,'Call AimsFunctionProjection -op 1 ...')
data_files = []
output_texs = []
p_ids = None
if bin_threshold is not None:
d,h = read_volume(fmri_data_file)
if np.allclose(d.astype(int), d):
tmp_dir = pyhrf.get_tmp_path()
p_ids = np.unique(d)
pyhrf.verbose(2, 'bin threshold: %f' %bin_threshold)
pyhrf.verbose(2, 'pids(n=%d): %d...%d' \
%(len(p_ids),min(p_ids),max(p_ids)))
for i,p_id in enumerate(p_ids):
if p_id != 0:
new_p = np.zeros_like(d)
new_p[np.where(d==p_id)] = i + 1 #0 is background
ifn = op.join(tmp_dir,'pmask_%d.nii'%p_id)
write_volume(new_p, ifn, h)
data_files.append(ifn)
ofn = op.join(tmp_dir,'ptex_%d.gii'%p_id)
output_texs.append(ofn)
else:
data_files.append(fmri_data_file)
output_texs.append(output_tex)
else:
data_files.append(fmri_data_file)
output_texs.append(output_tex)
pyhrf.verbose(3, 'input data files: %s' %str(data_files))
pyhrf.verbose(3, 'output data files: %s' %str(output_texs))
for data_file, o_tex in zip(data_files, output_texs):
projection = [
'AimsFunctionProjection',
'-op', '1',
'-d', kernels_file,
'-d1', data_file,
'-m', input_mesh,
'-o', o_tex
]
cmd = ' '.join(map(str,projection))
pyhrf.verbose(3, 'cmd: %s' %cmd)
os.system(cmd)
if bin_threshold is not None:
pyhrf.verbose(2, 'Binary threshold of texture at %f' %bin_threshold)
o_tex = output_texs[0]
data,data_gii = read_texture(o_tex)
data = (data>bin_threshold).astype(np.int32)
print 'data:', data.dtype
if p_ids is not None:
for pid, o_tex in zip(p_ids[1:], output_texs[1:]):
pdata,pdata_gii = read_texture(o_tex)
data += (pdata>bin_threshold).astype(np.int32) * pid
#assert (np.unique(data) == p_ids).all()
write_texture(data, output_tex, intent='NIFTI_INTENT_LABEL')
def setUp(self):
np.random.seed(8652761)
self.tmp_outputs = True #save outputs in tmp dir
#if False then save in current dir
if not self.tmp_outputs:
self.tmp_dir_small = './JDE_MS_test_small_simu'
if not op.exists(self.tmp_dir_small):
os.makedirs(self.tmp_dir_small)
self.tmp_dir_big = './JDE_MS_test_big_simu'
if not op.exists(self.tmp_dir_big): os.makedirs(self.tmp_dir_big)
else:
self.tmp_dir_small = pyhrf.get_tmp_path()
self.tmp_dir_big = pyhrf.get_tmp_path()
simu = simulate_sessions(output_dir=self.tmp_dir_small,
snr_scenario='high_snr',
spatial_size='tiny')
self.data_small_simu = merge_fmri_sessions(simu)
simu = simulate_sessions(output_dir=self.tmp_dir_big,
snr_scenario='low_snr',
spatial_size='normal')
self.data_simu = merge_fmri_sessions(simu)
# Parameters for multi-sessions sampler
dict_beta_single = {
BetaSampler.P_VAL_INI: np.array([0.5]),
BetaSampler.P_SAMPLE_FLAG: False,
BetaSampler.P_PARTITION_FUNCTION_METH: 'es',
BetaSampler.P_USE_TRUE_VALUE: False,
}
dict_hrf_single = {
HRF_MultiSess_Sampler.P_SAMPLE_FLAG: False,
HRF_MultiSess_Sampler.P_NORMALISE: 1., # normalise samples
HRF_MultiSess_Sampler.P_USE_TRUE_VALUE: True,
HRF_MultiSess_Sampler.P_ZERO_CONSTR: True,
#HRF_MultiSess_Sampler.P_PRIOR_TYPE : 'singleHRF',
}
dict_var_hrf_single = {
RHSampler.P_SAMPLE_FLAG: False,
RHSampler.P_VAL_INI: np.array([0.001]),
}
dict_nrl_sess_single = {
NRL_Multi_Sess_Sampler.P_SAMPLE_FLAG: False,
NRL_Multi_Sess_Sampler.P_USE_TRUE_VALUE: True,
}
dict_nrl_sess_var_single = {
Variance_GaussianNRL_Multi_Sess.P_SAMPLE_FLAG: False,
Variance_GaussianNRL_Multi_Sess.P_USE_TRUE_VALUE: True,
}
dict_nrl_bar_single = {
NRLsBar_Drift_Multi_Sess_Sampler.P_SAMPLE_FLAG: False,
NRLsBar_Drift_Multi_Sess_Sampler.P_USE_TRUE_NRLS: True,
NRLsBar_Drift_Multi_Sess_Sampler.P_SAMPLE_LABELS: False,
NRLsBar_Drift_Multi_Sess_Sampler.P_USE_TRUE_LABELS: True,
}
dict_drift_single = {
Drift_MultiSess_Sampler.P_SAMPLE_FLAG: False,
Drift_MultiSess_Sampler.P_USE_TRUE_VALUE: True,
}
dict_drift_var_single = {
ETASampler_MultiSess.P_SAMPLE_FLAG: False,
ETASampler_MultiSess.P_USE_TRUE_VALUE: True,
}
dict_noise_var_single = {
NoiseVariance_Drift_Multi_Sess_Sampler.P_SAMPLE_FLAG: False,
NoiseVariance_Drift_Multi_Sess_Sampler.P_USE_TRUE_VALUE: True,
}
dict_mixt_param_single = {
BiGaussMixtureParamsSampler.P_SAMPLE_FLAG: False,
BiGaussMixtureParamsSampler.P_USE_TRUE_VALUE: True,
BiGaussMixtureParamsSampler.P_HYPER_PRIOR: 'Jeffrey',
}
self.sampler_params_for_single_test = {
BMSS.P_NB_ITERATIONS:
100,
BMSS.P_SMPL_HIST_PACE:
-1,
BMSS.P_OBS_HIST_PACE:
-1,
# level of spatial correlation = beta
BMSS.P_BETA:
BetaSampler(dict_beta_single),
# HRF
BMSS.P_HRF:
HRF_MultiSess_Sampler(dict_hrf_single),
# HRF variance
BMSS.P_RH:
RHSampler(dict_var_hrf_single),
# neural response levels (stimulus-induced effects) by session
BMSS.P_NRLS_SESS:
NRL_Multi_Sess_Sampler(dict_nrl_sess_single),
# neural response levels by session --> variance
BMSS.P_NRLS_SESS_VAR:
Variance_GaussianNRL_Multi_Sess(dict_nrl_sess_var_single),
# neural response levels mean: over sessions
BMSS.P_NRLS_BAR:
NRLsBar_Drift_Multi_Sess_Sampler(dict_nrl_bar_single),
# drift
BMSS.P_DRIFT:
Drift_MultiSess_Sampler(dict_drift_single),
#drift variance
BMSS.P_ETA:
ETASampler_MultiSess(dict_drift_var_single),
#noise variance
BMSS.P_NOISE_VAR_SESS:
NoiseVariance_Drift_Multi_Sess_Sampler(dict_noise_var_single),
#weights o fthe mixture
#parameters of the mixture
BMSS.P_MIXT_PARAM_NRLS_BAR:
BiGaussMixtureParamsSampler(dict_mixt_param_single),
BMSS.P_CHECK_FINAL_VALUE:
'raise', #print or raise
}
# Parameters for multi-sessions sampler - full test
dict_beta_full = {
BetaSampler.P_VAL_INI: np.array([0.5]),
BetaSampler.P_SAMPLE_FLAG: True,
BetaSampler.P_PARTITION_FUNCTION_METH: 'es',
}
dict_hrf_full = {
HRF_MultiSess_Sampler.P_SAMPLE_FLAG: True,
HRF_MultiSess_Sampler.P_NORMALISE: 1., # normalise samples
HRF_MultiSess_Sampler.P_USE_TRUE_VALUE: False,
HRF_MultiSess_Sampler.P_ZERO_CONSTR: True,
#HRF_MultiSess_Sampler.P_PRIOR_TYPE : 'singleHRF',
}
dict_var_hrf_full = {
RHSampler.P_SAMPLE_FLAG: False,
RHSampler.P_VAL_INI: np.array([0.001]),
}
dict_nrl_sess_full = {
NRL_Multi_Sess_Sampler.P_SAMPLE_FLAG: True,
NRL_Multi_Sess_Sampler.P_USE_TRUE_VALUE: False,
}
dict_nrl_sess_var_full = {
Variance_GaussianNRL_Multi_Sess.P_SAMPLE_FLAG: True,
Variance_GaussianNRL_Multi_Sess.P_USE_TRUE_VALUE: False,
}
dict_nrl_bar_full = {
NRLsBar_Drift_Multi_Sess_Sampler.P_SAMPLE_FLAG: True,
NRLsBar_Drift_Multi_Sess_Sampler.P_USE_TRUE_NRLS: False,
NRLsBar_Drift_Multi_Sess_Sampler.P_SAMPLE_LABELS: True,
NRLsBar_Drift_Multi_Sess_Sampler.P_USE_TRUE_LABELS: False,
}
dict_drift_full = {
Drift_MultiSess_Sampler.P_SAMPLE_FLAG: True,
Drift_MultiSess_Sampler.P_USE_TRUE_VALUE: False,
}
dict_drift_var_full = {
ETASampler_MultiSess.P_SAMPLE_FLAG: True,
ETASampler_MultiSess.P_USE_TRUE_VALUE: False,
}
dict_noise_var_full = {
NoiseVariance_Drift_Multi_Sess_Sampler.P_SAMPLE_FLAG: True,
NoiseVariance_Drift_Multi_Sess_Sampler.P_USE_TRUE_VALUE: False,
}
dict_mixt_param_full = {
BiGaussMixtureParamsSampler.P_SAMPLE_FLAG: True,
BiGaussMixtureParamsSampler.P_USE_TRUE_VALUE: False,
BiGaussMixtureParamsSampler.P_HYPER_PRIOR: 'Jeffrey',
}
self.sampler_params_for_full_test = {
BMSS.P_NB_ITERATIONS:
400,
BMSS.P_SMPL_HIST_PACE:
-1,
BMSS.P_OBS_HIST_PACE:
-1,
# level of spatial correlation = beta
BMSS.P_BETA:
BetaSampler(dict_beta_full),
# HRF
BMSS.P_HRF:
HRF_MultiSess_Sampler(dict_hrf_full),
# HRF variance
BMSS.P_RH:
RHSampler(dict_var_hrf_full),
# neural response levels (stimulus-induced effects) by session
BMSS.P_NRLS_SESS:
NRL_Multi_Sess_Sampler(dict_nrl_sess_full),
# neural response levels by session --> variance
BMSS.P_NRLS_SESS_VAR:
Variance_GaussianNRL_Multi_Sess(dict_nrl_sess_var_full),
# neural response levels mean: over sessions
BMSS.P_NRLS_BAR:
NRLsBar_Drift_Multi_Sess_Sampler(dict_nrl_bar_full),
# drift
BMSS.P_DRIFT:
Drift_MultiSess_Sampler(dict_drift_full),
#drift variance
BMSS.P_ETA:
ETASampler_MultiSess(dict_drift_var_full),
#noise variance
BMSS.P_NOISE_VAR_SESS:
NoiseVariance_Drift_Multi_Sess_Sampler(dict_noise_var_full),
#weights o fthe mixture
#parameters of the mixture
BMSS.P_MIXT_PARAM_NRLS_BAR:
BiGaussMixtureParamsSampler(dict_mixt_param_full),
BMSS.P_CHECK_FINAL_VALUE:
'raise', #print or raise
}
# -*- coding: utf-8 -*-
#
"""
Compute the mean of BOLD signal within parcels.
This is an example of several operations for xndarray:
- explosion of data according to a parcellation mask
- mean over voxel
- merge of several xndarray objects
"""
import os.path as op
from pyhrf import get_data_file_name, get_tmp_path
from pyhrf.ndarray import xndarray, merge
func_data = xndarray.load(get_data_file_name('subj0_bold_session0.nii.gz'))
parcellation = xndarray.load(get_data_file_name('subj0_parcellation.nii.gz'))
parcel_fdata = func_data.explode(parcellation)
parcel_means = dict((parcel_id, d.copy().fill(d.mean('position')))
for parcel_id, d in parcel_fdata.items())
parcel_means = merge(parcel_means, parcellation, axis='position')
output_fn = op.join(get_tmp_path(), './subj0_bold_parcel_means.nii')
print 'File saved to:', output_fn
parcel_means.save(output_fn)
#TODO test full script
def run(self, parallel=None, n_jobs=None):
"""
Run the analysis: load data, run estimation, output results
"""
if parallel is None:
result = self.execute()
elif parallel == 'local':
cfg_parallel = pyhrf.cfg['parallel-local']
try:
from joblib import Parallel, delayed
except ImportError:
raise Exception(
'Can not import joblib. It is required to '
'enable parallel processing on a local machine.')
effective_level = logger.getEffectiveLevel()
if effective_level == logging.DEBUG:
parallel_verb = 11
elif effective_level == logging.INFO:
parallel_verb = 2
else:
parallel_verb = 0
if n_jobs is None:
if cfg_parallel["nb_procs"]:
n_jobs = cfg_parallel["nb_procs"]
else:
n_jobs = available_cpu_count()
p = Parallel(n_jobs=n_jobs, verbose=parallel_verb)
result = p(delayed(exec_t)(t) for t in self.split(output_dir=None))
# join list of lists:
result = list(itertools.chain.from_iterable(result))
elif parallel == 'LAN':
from pyhrf import grid
cfg_parallel = pyhrf.cfg['parallel-LAN']
remoteUser = cfg_parallel['user']
# 1. Some checks on input/output directory
remoteDir = cfg_parallel['remote_path']
# At the end, results will be retrieved direclty from remoteDir,
# which has to be readable
if remoteDir is None or not op.exists(remoteDir):
raise Exception('Remote directory is not readable (%s).'
'Consider mounting it with sshfs.' % remoteDir)
# Try if remoteDir is writeable, so that we don't need to upload
# data via ssh
remote_writeable = False
if os.access(remoteDir, os.W_OK):
remote_writeable = True
tmpDir = remoteDir
else:
logger.info('Remote dir is not writeable -> using tmp '
'dir to store splitted data & then upload.')
# 2. split roi data
logger.info('Path to store sub treatments: %s', tmpDir)
treatments_dump_files = []
self.split(dump_sub_results=True,
output_dir=tmpDir,
make_sub_outputs=False,
output_file_list=treatments_dump_files)
# 3. copy data to remote directory
if not remote_writeable:
host = cfg_parallel['remote_host']
logger.info('Uploading data to %s ...', remoteDir)
remote_input_files = remote_copy(treatments_dump_files, host,
remoteUser, remoteDir)
# 4. create job list
tasks_list = []
for f in treatments_dump_files:
f = op.join(remoteDir, op.basename(f))
nice = cfg_parallel['niceness']
tasks_list.append(
'nice -n %d %s -v%d -t "%s"' %
(nice, 'pyhrf_jde_estim', logger.getEffectiveLevel(), f))
mode = 'dispatch'
tasks = grid.read_tasks(';'.join(tasks_list), mode)
timeslot = grid.read_timeslot('allday')
hosts = grid.read_hosts(cfg_parallel['hosts'])
if self.output_dir is not None:
brokenfile = op.join(self.output_dir, 'pyhrf-broken_cmd.batch')
logfile = op.join(self.output_dir, 'pyhrf-parallel.log')
logger.info('Log file for process dispatching: %s', logfile)
else:
brokenfile = None
logfile = None
# 3. launch them
logger.info('Dispatching processes ...')
try:
grid.run_grid(mode,
hosts,
'rsa',
tasks,
timeslot,
brokenfile,
logfile,
user=remoteUser)
grid.kill_threads()
except KeyboardInterrupt:
grid.quit(None, None)
if brokenfile is not None and len(
open(brokenfile).readlines()) > 0:
logger.info('There are some broken commands, trying again ...')
try:
tasks = grid.read_tasks(brokenfile, mode)
grid.run_grid(mode,
hosts,
'rsa',
tasks,
timeslot,
brokenfile,
logfile,
user=remoteUser)
grid.kill_threads()
except KeyboardInterrupt:
grid.quit(None, None)
# 3.1 grab everything back ??
# try:
# "scp %s@%s:%s %s" %(remoteUser,host,
# op.join(remoteDir,'result*'),
# op.abspath(op.dirname(options.cfgFile))))
# TODO : test if everything went fine
# 4. merge all results and create outputs
result = []
# if op.exists(remoteDir): TODO :scp if remoteDir not readable
nb_treatments = len(treatments_dump_files)
remote_result_files = [
op.join(remoteDir, 'result_%04d.pck' % i)
for i in range(nb_treatments)
]
logger.info('remote_result_files: %s', str(remote_result_files))
nres = len(filter(op.exists, remote_result_files))
if nres == nb_treatments:
logger.info('Grabbing results ...')
for fnresult in remote_result_files:
fresult = open(fnresult)
result.append(cPickle.load(fresult)[0])
fresult.close()
else:
print 'Found only %d result files (expected %d)' \
% (nres, nb_treatments)
print 'Something went wrong, check the log files'
if not remote_writeable:
logger.info('Cleaning tmp dir (%s)...', tmpDir)
shutil.rmtree(tmpDir)
logger.info('Cleaning up remote dir (%s) through ssh ...',
remoteDir)
cmd = 'ssh %s@%s rm -f "%s" "%s" ' \
% (remoteUser, host, ' '.join(remote_result_files),
' '.join(remote_input_files))
logger.info(cmd)
os.system(cmd)
else:
if 0:
logger.info('Cleaning up remote dir (%s)...', remoteDir)
for f in os.listdir(remoteDir):
os.remove(op.join(remoteDir, f))
elif parallel == 'cluster':
from pyhrf.parallel import run_soma_workflow
cfg = pyhrf.cfg['parallel-cluster']
# create tmp remote path:
date_now = time.strftime('%c').replace(' ', '_').replace(':', '_')
remote_path = op.join(cfg['remote_path'], date_now)
logger.info('Create tmp remote dir: %s', remote_path)
remote_mkdir(cfg['server'], cfg['user'], remote_path)
t_name = 'default_treatment'
tmp_dir = pyhrf.get_tmp_path()
label_for_cluster = self.analyser.get_label()
if self.output_dir is None:
out_dir = pyhrf.get_tmp_path()
else:
out_dir = self.output_dir
result = run_soma_workflow({t_name: self},
'pyhrf_jde_estim', {t_name: tmp_dir},
cfg['server_id'],
cfg['server'],
cfg['user'], {t_name: remote_path},
{t_name: op.abspath(out_dir)},
label_for_cluster,
wait_ending=True)
else:
raise Exception('Parallel mode "%s" not available' % parallel)
logger.info('Retrieved %d results', len(result))
return self.output(result, (self.result_dump_file is not None),
self.make_outputs)
def setUp(self):
self.tmp_dir = pyhrf.get_tmp_path() #'./'
def setUp(self):
# called before any unit test of the class
self.tmp_path = pyhrf.get_tmp_path() # create a temporary folder
self.clean_tmp = True
def setUp(self):
#pyhrf.verbose.set_verbosity(2)
np.random.seed(8652761)
# tmpDir = tempfile.mkdtemp(prefix='pyhrf_tests',
# dir=pyhrf.cfg['global']['tmp_path'])
self.tmp_outputs = True #save outputs in tmp dir
#if False then save in current dir
if not self.tmp_outputs:
self.tmp_dir_small = './JDE_MS_test_small_simu'
if not op.exists(self.tmp_dir_small): os.makedirs(self.tmp_dir_small)
self.tmp_dir_big = './JDE_MS_test_big_simu'
if not op.exists(self.tmp_dir_big): os.makedirs(self.tmp_dir_big)
else:
self.tmp_dir_small = pyhrf.get_tmp_path()
self.tmp_dir_big = pyhrf.get_tmp_path()
simu = simulate_sessions(output_dir = self.tmp_dir_small,
snr_scenario='high_snr', spatial_size='tiny')
self.data_small_simu = merge_fmri_sessions(simu)
simu = simulate_sessions(output_dir=self.tmp_dir_big,
snr_scenario='low_snr', spatial_size='normal')
self.data_simu = merge_fmri_sessions(simu)
# Parameters for multi-sessions sampler
dict_beta_single = {
BetaSampler.P_VAL_INI : np.array([0.5]),
BetaSampler.P_SAMPLE_FLAG : False,
BetaSampler.P_PARTITION_FUNCTION_METH : 'es',
BetaSampler.P_USE_TRUE_VALUE : False,
}
dict_hrf_single = {
HRF_MultiSess_Sampler.P_SAMPLE_FLAG : False,
HRF_MultiSess_Sampler.P_NORMALISE : 1., # normalise samples
HRF_MultiSess_Sampler.P_USE_TRUE_VALUE : True,
HRF_MultiSess_Sampler.P_ZERO_CONSTR : True,
#HRF_MultiSess_Sampler.P_PRIOR_TYPE : 'singleHRF',
}
dict_var_hrf_single = {
RHSampler.P_SAMPLE_FLAG : False,
RHSampler.P_VAL_INI : np.array([0.001]),
}
dict_nrl_sess_single = {
NRL_Multi_Sess_Sampler.P_SAMPLE_FLAG : False,
NRL_Multi_Sess_Sampler.P_USE_TRUE_VALUE : True,
}
dict_nrl_sess_var_single = {
Variance_GaussianNRL_Multi_Sess.P_SAMPLE_FLAG : False,
Variance_GaussianNRL_Multi_Sess.P_USE_TRUE_VALUE : True,
}
dict_nrl_bar_single = {
NRLsBar_Drift_Multi_Sess_Sampler.P_SAMPLE_FLAG : False,
NRLsBar_Drift_Multi_Sess_Sampler.P_USE_TRUE_NRLS : True,
NRLsBar_Drift_Multi_Sess_Sampler.P_SAMPLE_LABELS : False,
NRLsBar_Drift_Multi_Sess_Sampler.P_USE_TRUE_LABELS : True,
}
dict_drift_single = {
Drift_MultiSess_Sampler.P_SAMPLE_FLAG : False,
Drift_MultiSess_Sampler.P_USE_TRUE_VALUE : True,
}
dict_drift_var_single = {
ETASampler_MultiSess.P_SAMPLE_FLAG : False,
ETASampler_MultiSess.P_USE_TRUE_VALUE : True,
}
dict_noise_var_single = {
NoiseVariance_Drift_Multi_Sess_Sampler.P_SAMPLE_FLAG : False,
NoiseVariance_Drift_Multi_Sess_Sampler.P_USE_TRUE_VALUE : True,
}
dict_mixt_param_single = {
BiGaussMixtureParamsSampler.P_SAMPLE_FLAG : False,
BiGaussMixtureParamsSampler.P_USE_TRUE_VALUE : True,
BiGaussMixtureParamsSampler.P_HYPER_PRIOR : 'Jeffrey',
}
self.sampler_params_for_single_test = {
BMSS.P_NB_ITERATIONS : 100,
BMSS.P_SMPL_HIST_PACE : -1,
BMSS.P_OBS_HIST_PACE : -1,
# level of spatial correlation = beta
BMSS.P_BETA : BetaSampler(dict_beta_single),
# HRF
BMSS.P_HRF : HRF_MultiSess_Sampler(dict_hrf_single),
# HRF variance
BMSS.P_RH : RHSampler(dict_var_hrf_single),
# neural response levels (stimulus-induced effects) by session
BMSS.P_NRLS_SESS : NRL_Multi_Sess_Sampler(dict_nrl_sess_single),
# neural response levels by session --> variance
BMSS.P_NRLS_SESS_VAR : Variance_GaussianNRL_Multi_Sess(dict_nrl_sess_var_single),
# neural response levels mean: over sessions
BMSS.P_NRLS_BAR : NRLsBar_Drift_Multi_Sess_Sampler(dict_nrl_bar_single),
# drift
BMSS.P_DRIFT : Drift_MultiSess_Sampler(dict_drift_single),
#drift variance
BMSS.P_ETA : ETASampler_MultiSess(dict_drift_var_single),
#noise variance
BMSS.P_NOISE_VAR_SESS : NoiseVariance_Drift_Multi_Sess_Sampler(dict_noise_var_single),
#weights o fthe mixture
#parameters of the mixture
BMSS.P_MIXT_PARAM_NRLS_BAR : BiGaussMixtureParamsSampler(dict_mixt_param_single),
BMSS.P_CHECK_FINAL_VALUE : 'raise', #print or raise
}
# Parameters for multi-sessions sampler - full test
dict_beta_full = {
BetaSampler.P_VAL_INI : np.array([0.5]),
BetaSampler.P_SAMPLE_FLAG : True,
BetaSampler.P_PARTITION_FUNCTION_METH : 'es',
}
dict_hrf_full = {
HRF_MultiSess_Sampler.P_SAMPLE_FLAG : True,
HRF_MultiSess_Sampler.P_NORMALISE : 1., # normalise samples
HRF_MultiSess_Sampler.P_USE_TRUE_VALUE : False,
HRF_MultiSess_Sampler.P_ZERO_CONSTR : True,
#HRF_MultiSess_Sampler.P_PRIOR_TYPE : 'singleHRF',
}
dict_var_hrf_full = {
RHSampler.P_SAMPLE_FLAG : False,
RHSampler.P_VAL_INI : np.array([0.001]),
}
dict_nrl_sess_full = {
NRL_Multi_Sess_Sampler.P_SAMPLE_FLAG : True,
NRL_Multi_Sess_Sampler.P_USE_TRUE_VALUE : False,
}
dict_nrl_sess_var_full = {
Variance_GaussianNRL_Multi_Sess.P_SAMPLE_FLAG : True,
Variance_GaussianNRL_Multi_Sess.P_USE_TRUE_VALUE : False,
}
dict_nrl_bar_full = {
NRLsBar_Drift_Multi_Sess_Sampler.P_SAMPLE_FLAG : True,
NRLsBar_Drift_Multi_Sess_Sampler.P_USE_TRUE_NRLS : False,
NRLsBar_Drift_Multi_Sess_Sampler.P_SAMPLE_LABELS : True,
NRLsBar_Drift_Multi_Sess_Sampler.P_USE_TRUE_LABELS : False,
}
dict_drift_full = {
Drift_MultiSess_Sampler.P_SAMPLE_FLAG : True,
Drift_MultiSess_Sampler.P_USE_TRUE_VALUE : False,
}
dict_drift_var_full = {
ETASampler_MultiSess.P_SAMPLE_FLAG : True,
ETASampler_MultiSess.P_USE_TRUE_VALUE : False,
}
dict_noise_var_full = {
NoiseVariance_Drift_Multi_Sess_Sampler.P_SAMPLE_FLAG : True,
NoiseVariance_Drift_Multi_Sess_Sampler.P_USE_TRUE_VALUE : False,
}
dict_mixt_param_full = {
BiGaussMixtureParamsSampler.P_SAMPLE_FLAG : True,
BiGaussMixtureParamsSampler.P_USE_TRUE_VALUE : False,
BiGaussMixtureParamsSampler.P_HYPER_PRIOR : 'Jeffrey',
}
self.sampler_params_for_full_test = {
BMSS.P_NB_ITERATIONS : 400,
BMSS.P_SMPL_HIST_PACE : -1,
BMSS.P_OBS_HIST_PACE : -1,
# level of spatial correlation = beta
BMSS.P_BETA : BetaSampler(dict_beta_full),
# HRF
BMSS.P_HRF : HRF_MultiSess_Sampler(dict_hrf_full),
# HRF variance
BMSS.P_RH : RHSampler(dict_var_hrf_full),
# neural response levels (stimulus-induced effects) by session
BMSS.P_NRLS_SESS : NRL_Multi_Sess_Sampler(dict_nrl_sess_full),
# neural response levels by session --> variance
BMSS.P_NRLS_SESS_VAR : Variance_GaussianNRL_Multi_Sess(dict_nrl_sess_var_full),
# neural response levels mean: over sessions
BMSS.P_NRLS_BAR : NRLsBar_Drift_Multi_Sess_Sampler(dict_nrl_bar_full),
# drift
BMSS.P_DRIFT : Drift_MultiSess_Sampler(dict_drift_full),
#drift variance
BMSS.P_ETA : ETASampler_MultiSess(dict_drift_var_full),
#noise variance
BMSS.P_NOISE_VAR_SESS : NoiseVariance_Drift_Multi_Sess_Sampler(dict_noise_var_full),
#weights o fthe mixture
#parameters of the mixture
BMSS.P_MIXT_PARAM_NRLS_BAR : BiGaussMixtureParamsSampler(dict_mixt_param_full),
BMSS.P_CHECK_FINAL_VALUE : 'raise', #print or raise
}
def setUp(self):
np.random.seed(8652761)
self.tmp_dir = pyhrf.get_tmp_path()
self.clean_tmp = False # HACK True
#
"""
Compute the mean of BOLD signal within parcels.
This is an example of several operations for xndarray:
- explosion of data according to a parcellation mask
- mean over voxel
- merge of several xndarray objects
"""
import os.path as op
from pyhrf import get_data_file_name, get_tmp_path
from pyhrf.ndarray import xndarray, merge
func_data = xndarray.load(get_data_file_name("subj0_bold_session0.nii.gz"))
parcellation = xndarray.load(get_data_file_name("subj0_parcellation.nii.gz"))
parcel_fdata = func_data.explode(parcellation)
parcel_means = dict((parcel_id, d.copy().fill(d.mean("position"))) for parcel_id, d in parcel_fdata.items())
parcel_means = merge(parcel_means, parcellation, axis="position")
output_fn = op.join(get_tmp_path(), "./subj0_bold_parcel_means.nii")
print "File saved to:", output_fn
parcel_means.save(output_fn)
# TODO test full script
def setUp(self):
pyhrf.verbose.set_verbosity(0)
np.random.seed(8652761)
#self.simu_dir = './simulation'
#if not op.exists(self.simu_dir): os.makedirs(self.simu_dir)
self.simu_dir = pyhrf.get_tmp_path()
# Parameters to setup a Sampler where all samplers are OFF and
# set to their true values.
# This is used by the function _test_specific_samplers,
# which will turn on specific samplers to test.
self.sampler_params_for_single_test = {
BMSS.P_NB_ITERATIONS : 40,
BMSS.P_SMPL_HIST_PACE : -1,
BMSS.P_OBS_HIST_PACE : -1,
# level of spatial correlation = beta
#BMSS.P_BETA : BetaSampler(dict_beta_single),
# HRF by subject
BMSS.P_HRF_SUBJ : jms.HRF_Sampler(do_sampling=False,
normalise=1.,
use_true_value=True,
zero_contraint=False,
prior_type='singleHRF'),
# HRF variance
BMSS.P_RH_SUBJ : jms.HRFVarianceSubjectSampler(do_sampling=False,
use_true_value=True),
# HRF group
BMSS.P_HRF_GROUP : jms.HRF_Group_Sampler(do_sampling=False,
normalise=1.,
use_true_value=True,
zero_contraint=False,
prior_type='singleHRF'),
# HRF variance
BMSS.P_RH_GROUP : jms.RHGroupSampler(do_sampling=False,
use_true_value=True),
# neural response levels (stimulus-induced effects) by subject
BMSS.P_NRLS_SUBJ : jms.NRLs_Sampler(do_sampling=False,
use_true_value=True),
BMSS.P_LABELS : jms.LabelSampler(do_sampling=False,
use_true_value=True),
# drift
BMSS.P_DRIFT : jms.Drift_MultiSubj_Sampler(do_sampling=False,
use_true_value=True),
#drift variance
BMSS.P_ETA : jms.ETASampler_MultiSubj(do_sampling=False,
use_true_value=True),
#noise variance
BMSS.P_NOISE_VAR_SUBJ : \
jms.NoiseVariance_Drift_MultiSubj_Sampler(do_sampling=False,
use_true_value=False),
#weights o fthe mixture
#parameters of the mixture
BMSS.P_MIXT_PARAM_NRLS : jms.MixtureParamsSampler(do_sampling=False,
use_true_value=False),
#BMSS.P_ALPHA_SUBJ : Alpha_hgroup_Sampler(dict_alpha_single),
#BMSS.P_ALPHA_VAR_SUBJ : AlphaVar_Sampler(dict_alpha_var_single),
BMSS.P_CHECK_FINAL_VALUE : 'none', #print or raise
}
def setUp(self):
self.tmp_dir = pyhrf.get_tmp_path()
from pyhrf import get_tmp_path
import numpy as np
import pyhrf.boldsynth.scenarios as simu
from pyhrf.tools import Pipeline
simulation_steps = {
'dt' : 0.6,
'dsf' : 4, #downsampling factor -> tr = dt * dsf = 2.4
'mask' : np.array([[[1,1,1,1,1,1,1]]]),
'labels' : np.array([[0,0,1,1,0,1,1]]),
'mean_act' : 3.,
'var_act' : 0.5,
'mean_inact' : 0.,
'var_inact' : 0.5,
'nrls' : simu.create_bigaussian_nrls,
'rastered_paradigm' : np.array([[0,0,1,0,0,0,1,0]]),
'hrf' : simu.create_canonical_hrf,
'v_noise' : 1.,
'bold_shape' : simu.get_bold_shape,
'noise' : simu.create_gaussian_noise,
'stim_induced_signal' : simu.create_stim_induced_signal,
'bold' : simu.create_bold,
}
simulation = Pipeline(simulation_steps)
simulation.resolve()
simulation_items = simulation.get_values()
output_dir = get_tmp_path()
print 'Save simulation to:', output_dir
simu.save_simulation(simulation_items, output_dir=output_dir)
def setUp(self):
# called before any unit test of the class
self.tmp_path = pyhrf.get_tmp_path() # create a temporary folder
self.clean_tmp = True
def project_fmri_from_kernels(
input_mesh,
kernels_file,
fmri_data_file,
output_tex,
bin_threshold=None,
):
logger.info('Project data onto mesh using kernels ...')
if 0:
print 'Projecting ...'
print 'func data:', fmri_data_file
print 'Mesh file:', input_mesh
print 'Save as:', output_tex
logger.info('Call AimsFunctionProjection -op 1 ...')
data_files = []
output_texs = []
p_ids = None
if bin_threshold is not None:
d, h = read_volume(fmri_data_file)
if np.allclose(d.astype(int), d):
tmp_dir = pyhrf.get_tmp_path()
p_ids = np.unique(d)
logger.info('bin threshold: %f', bin_threshold)
logger.info('pids(n=%d): %d...%d', len(p_ids), min(p_ids),
max(p_ids))
for i, p_id in enumerate(p_ids):
if p_id != 0:
new_p = np.zeros_like(d)
new_p[np.where(d == p_id)] = i + 1 # 0 is background
ifn = op.join(tmp_dir, 'pmask_%d.nii' % p_id)
write_volume(new_p, ifn, h)
data_files.append(ifn)
ofn = op.join(tmp_dir, 'ptex_%d.gii' % p_id)
output_texs.append(ofn)
else:
data_files.append(fmri_data_file)
output_texs.append(output_tex)
else:
data_files.append(fmri_data_file)
output_texs.append(output_tex)
logger.info('input data files: %s', str(data_files))
logger.info('output data files: %s', str(output_texs))
for data_file, o_tex in zip(data_files, output_texs):
projection = [
'AimsFunctionProjection', '-op', '1', '-d', kernels_file, '-d1',
data_file, '-m', input_mesh, '-o', o_tex
]
cmd = ' '.join(map(str, projection))
logger.info('cmd: %s', cmd)
os.system(cmd)
if bin_threshold is not None:
logger.info('Binary threshold of texture at %f', bin_threshold)
o_tex = output_texs[0]
data, data_gii = read_texture(o_tex)
data = (data > bin_threshold).astype(np.int32)
print 'data:', data.dtype
if p_ids is not None:
for pid, o_tex in zip(p_ids[1:], output_texs[1:]):
pdata, pdata_gii = read_texture(o_tex)
data += (pdata > bin_threshold).astype(np.int32) * pid
#assert (np.unique(data) == p_ids).all()
write_texture(data, output_tex, intent='NIFTI_INTENT_LABEL')
def run(self, parallel=None, n_jobs=None):
"""
Run the the analysis: load data, run estimation, output results
"""
if parallel is None:
result = self.execute()
elif parallel == 'local':
cfg_parallel = pyhrf.cfg['parallel-local']
try:
from joblib import Parallel, delayed
except ImportError:
raise Exception('Can not import joblib. It is required to '\
'enable parallel processing on a local machine.')
parallel_verb = pyhrf.verbose.verbosity
if pyhrf.verbose.verbosity == 6:
parallel_verb = 10
if n_jobs is None:
n_jobs = cfg_parallel['nb_procs']
p = Parallel(n_jobs=n_jobs, verbose=parallel_verb)
result = p(delayed(exec_t)(t) for t in self.split(output_dir=None))
# join list of lists:
result = list(itertools.chain.from_iterable(result))
elif parallel == 'LAN':
from pyhrf import grid
cfg_parallel = pyhrf.cfg['parallel-LAN']
remoteUser = cfg_parallel['user']
#1. Some checks on input/output directory
remoteDir = cfg_parallel['remote_path']
# At the end, results will be retrieved direclty from remoteDir,
# which has to be readable
if remoteDir is None or not op.exists(remoteDir):
raise Exception('Remote directory is not readable (%s).' \
'Consider mounting it with sshfs.'
%remoteDir)
# Try if remoteDir is writeable, so that we don't need to upload
# data via ssh
remote_writeable = False
if os.access(remoteDir, os.W_OK):
remote_writeable = True
tmpDir = remoteDir
else:
pyhrf.verbose(1, 'Remote dir is not writeable -> using tmp ' \
'dir to store splitted data & then upload.')
#2. split roi data
pyhrf.verbose(1, 'Path to store sub treatments: %s' %tmpDir)
treatments_dump_files = []
self.split(dump_sub_results=True, output_dir=tmpDir,
make_sub_outputs=False,
output_file_list=treatments_dump_files)
#3. copy data to remote directory
if not remote_writeable:
host = cfg_parallel['remote_host']
pyhrf.verbose(1, 'Uploading data to %s ...' %(remoteDir))
remote_input_files = remote_copy(treatments_dump_files,
host, remoteUser, remoteDir)
#4. create job list
tasks_list = []
for f in treatments_dump_files:
f = op.join(remoteDir,op.basename(f))
nice = cfg_parallel['niceness']
tasks_list.append('nice -n %d %s -v%d -t "%s"' \
%(nice,'pyhrf_jde_estim',
pyhrf.verbose.verbosity,f))
mode = 'dispatch'
tasks = grid.read_tasks(';'.join(tasks_list), mode)
timeslot = grid.read_timeslot('allday')
hosts = grid.read_hosts(cfg_parallel['hosts'])
if self.output_dir is not None:
brokenfile = op.join(self.output_dir, 'pyhrf-broken_cmd.batch')
logfile = op.join(self.output_dir, 'pyhrf-parallel.log')
pyhrf.verbose(1, 'Log file for process dispatching: %s' \
%logfile)
else:
brokenfile = None
logfile = None
#3. launch them
pyhrf.verbose(1, 'Dispatching processes ...')
try:
grid.run_grid(mode, hosts, 'rsa', tasks, timeslot, brokenfile,
logfile, user=remoteUser)
grid.kill_threads()
except KeyboardInterrupt:
grid.quit(None, None)
if brokenfile is not None and len(open(brokenfile).readlines()) > 0:
pyhrf.verbose(1, 'There are some broken commands, '\
'trying again ...')
try:
tasks = grid.read_tasks(brokenfile, mode)
grid.run_grid(mode, hosts, 'rsa', tasks, timeslot, brokenfile,
logfile, user=remoteUser)
grid.kill_threads()
except KeyboardInterrupt:
grid.quit(None, None)
#3.1 grab everything back ??
#try:
# "scp %s@%s:%s %s" %(remoteUser,host,
# op.join(remoteDir,'result*'),
# op.abspath(op.dirname(options.cfgFile))))
#TODO : test if everything went fine
#4. merge all results and create outputs
result = []
#if op.exists(remoteDir): TODO :scp if remoteDir not readable
nb_treatments = len(treatments_dump_files)
remote_result_files = [op.join(remoteDir, 'result_%04d.pck' %i) \
for i in range(nb_treatments)]
pyhrf.verbose(1,'remote_result_files: %s' %str(remote_result_files))
nres = len(filter(op.exists,remote_result_files))
if nres == nb_treatments:
pyhrf.verbose(1, 'Grabbing results ...')
for fnresult in remote_result_files:
fresult = open(fnresult)
result.append(cPickle.load(fresult)[0])
fresult.close()
else:
print 'Found only %d result files (expected %d)' \
%(nres, nb_treatments)
print 'Something went wrong, check the log files'
if not remote_writeable:
pyhrf.verbose(1, 'Cleaning tmp dir (%s)...' %tmpDir)
shutil.rmtree(tmpDir)
pyhrf.verbose(1, 'Cleaning up remote dir (%s) through ssh ...' \
%remoteDir)
cmd = 'ssh %s@%s rm -f "%s" "%s" ' \
%(remoteUser, host, ' '.join(remote_result_files),
' '.join(remote_input_files))
pyhrf.verbose(2, cmd)
os.system(cmd)
else:
if 0:
pyhrf.verbose(1, 'Cleaning up remote dir (%s)...' %remoteDir)
for f in os.listdir(remoteDir):
os.remove(op.join(remoteDir,f))
elif parallel == 'cluster':
from pyhrf.parallel import run_soma_workflow
cfg = pyhrf.cfg['parallel-cluster']
#create tmp remote path:
date_now = time.strftime('%c').replace(' ','_').replace(':','_')
remote_path = op.join(cfg['remote_path'], date_now)
pyhrf.verbose(1,'Create tmp remote dir: %s' %remote_path)
remote_mkdir(cfg['server'], cfg['user'], remote_path)
#if self.result_dump_file
t_name = 'default_treatment'
tmp_dir = pyhrf.get_tmp_path()
label_for_cluster = self.analyser.get_label()
if self.output_dir is None:
out_dir = pyhrf.get_tmp_path()
else:
out_dir = self.output_dir
result = run_soma_workflow({t_name:self}, 'pyhrf_jde_estim',
{t_name:tmp_dir}, cfg['server_id'],
cfg['server'], cfg['user'],
{t_name:remote_path},
{t_name:op.abspath(out_dir)},
label_for_cluster, wait_ending=True)
else:
raise Exception('Parallel mode "%s" not available' %parallel)
pyhrf.verbose(1, 'Retrieved %d results' %len(result))
return self.output(result, (self.result_dump_file is not None),
self.make_outputs)
from pyhrf import get_tmp_path
import numpy as np
import pyhrf.boldsynth.scenarios as simu
from pyhrf.tools import Pipeline
simulation_steps = {
'dt' : 0.6,
'dsf' : 4, #downsampling factor -> tr = dt * dsf = 2.4
'mask' : np.array([[[1,1,1,1,1,1,1]]]),
'labels' : np.array([[0,0,1,1,0,1,1]]),
'mean_act' : 3.,
'var_act' : 0.5,
'mean_inact' : 0.,
'var_inact' : 0.5,
'nrls' : simu.create_bigaussian_nrls,
'rastered_paradigm' : np.array([[0,0,1,0,0,0,1,0]]),
'hrf' : simu.create_canonical_hrf,
'v_noise' : 1.,
'bold_shape' : simu.get_bold_shape,
'noise' : simu.create_gaussian_noise,
'stim_induced_signal' : simu.create_stim_induced_signal,
'bold' : simu.create_bold,
}
simulation = Pipeline(simulation_steps)
simulation.resolve()
simulation_items = simulation.get_values()
output_dir = get_tmp_path()
print 'Save simulation to:', output_dir
simu.save_simulation(simulation_items, output_dir=output_dir)
