def run_function_in_parallel(func,
all_params,
n_jobs,
split_jobs=True,
verbose=True):
# Filter our items that should not run (all input files should exist, and all output files shoud not)
params = [
p for p in all_params
if all(_run_func_in_parallel((func, [p], True, 0, verbose)))
]
print('*** Run {}/{} records, {} jobs ***'.format(len(params),
len(all_params), n_jobs))
ret = input('ok? (y/n) ')
if not au.is_true(ret):
return
if split_jobs:
chunks_indices = np.array_split(np.arange(len(params)), n_jobs)
chunk_params = [
(func, [params[ind]
for ind in chunk_indices], False, thread_ind, verbose)
for thread_ind, chunk_indices in enumerate(chunks_indices)
]
else:
chunk_params = [(func, [p], False, thread_ind, verbose)
for thread_ind, p in enumerate(params)]
utils.run_parallel(_run_func_in_parallel, chunk_params, n_jobs)
def convert_ct_to_mgz(subject,
ct_raw_input_fol,
ct_fol='',
output_name='ct_org.mgz',
overwrite=False,
print_only=False,
ask_before=False):
if not op.isdir(ct_fol):
ct_fol = utils.make_dir(op.join(MMVT_DIR, subject, 'ct'))
if op.isfile(op.join(ct_fol, 'ct_reg_to_mr.mgz')) and not overwrite:
return True
if op.isfile(op.join(ct_fol, output_name)):
return True
ct_fname = utils.select_one_file(glob.glob(op.join(ct_fol, '*.mgz')))
if op.isfile(ct_fname):
if utils.namebase(ct_fname) != 'ct_org':
utils.make_link(ct_fname, op.join(ct_fol, 'ct_org.mgz'))
return True
output_fname = op.join(ct_fol, output_name)
if op.isfile(output_fname):
if not overwrite:
return True
else:
os.remove(output_fname)
if op.isfile(op.join(SUBJECTS_DIR, subject, 'ct', 'ct.nii.gz')):
ct_files = [op.join(SUBJECTS_DIR, subject, 'ct', 'ct.nii.gz')]
elif op.isfile(op.join(SUBJECTS_DIR, subject, 'mri', 'ct.nii.gz')):
ct_files = [op.join(SUBJECTS_DIR, subject, 'mri', 'ct.nii.gz')]
else:
if not op.isdir(ct_raw_input_fol):
print('{} does not exist!'.format(ct_fol))
return False
ct_files = glob.glob(op.join(ct_raw_input_fol, '*.dcm'))
if len(ct_files) == 0:
sub_folders = [
d for d in glob.glob(op.join(ct_raw_input_fol, '*'))
if op.isdir(d)
]
if len(sub_folders) == 0:
print('Cannot find CT files in {}!'.format(ct_raw_input_fol))
return False
fol = utils.select_one_file(sub_folders, '', 'CT', is_dir=True)
ct_files = glob.glob(op.join(fol, '*.dcm'))
if len(ct_files) == 0:
print('Cannot find CT files in {}!'.format(fol))
return False
ct_files.sort(key=op.getmtime)
if ask_before:
ret = input('convert {} to {}? '.format(ct_files[0], output_fname))
if not au.is_true(ret):
return False
fu.mri_convert(ct_files[0], output_fname, print_only=print_only)
return True if print_only else op.isfile(output_fname)
def run_on_subjects(args, main_func, subjects_itr=None, subject_func=None):
if subjects_itr is None:
subjects_itr = args.subject
subjects_flags, subjects_errors = {}, {}
args.n_jobs = utils.get_n_jobs(args.n_jobs)
args.sftp_password = utils.get_sftp_password(
args.subject, SUBJECTS_DIR, args.necessary_files, args.sftp_username, args.overwrite_fs_files) \
if args.sftp else ''
if '*' in args.subject:
args.subject = [utils.namebase(fol) for fol in glob.glob(op.join(SUBJECTS_DIR, args.subject))]
os.environ['SUBJECTS_DIR'] = SUBJECTS_DIR
for tup in subjects_itr:
subject = get_subject(tup, subject_func)
utils.make_dir(op.join(MMVT_DIR, subject, 'mmvt'))
remote_subject_dir = utils.build_remote_subject_dir(args.remote_subject_dir, subject)
print('****************************************************************')
print('subject: {}, atlas: {}'.format(subject, args.atlas))
print('remote dir: {}'.format(remote_subject_dir))
print('****************************************************************')
os.environ['SUBJECT'] = subject
flags = dict()
if args.necessary_files == '':
args.necessary_files = dict()
try:
if utils.should_run(args, 'prepare_local_subjects_folder'):
# *) Prepare the local subject's folder
flags['prepare_local_subjects_folder'] = prepare_local_subjects_folder(
subject, remote_subject_dir, args)
if not flags['prepare_local_subjects_folder'] and not args.ignore_missing:
ans = input('Do you which to continue (y/n)? ')
if not au.is_true(ans):
continue
flags = main_func(tup, remote_subject_dir, args, flags)
subjects_flags[subject] = flags
except:
subjects_errors[subject] = traceback.format_exc()
print('Error in subject {}'.format(subject))
print(traceback.format_exc())
errors = defaultdict(list)
for subject, flags in subjects_flags.items():
print('subject {}:'.format(subject))
for flag_type, val in flags.items():
print('{}: {}'.format(flag_type, val))
if not val:
errors[subject].append(flag_type)
if len(errors) > 0:
print('Errors:')
for subject, error in errors.items():
print('{}: {}'.format(subject, error))
def recon_all_clin(args):
# python -m src.preproc.examples.anatomy -s nmr01426 -f recon_all_clin --clin_fol clin_6966926 --dicoms_fol Prisma_fit-67026-20200618-141203-000586
import os
for subject, clin_fol, dicoms_fol in zip(args.subject, args.clin_fol,
args.dicoms_fol):
clin_full_fol = utils.make_dir(
op.join(args.clin_root, clin_fol, 'mne_dicom'))
memprage_fols = glob.glob(op.join(clin_full_fol, '*MEMPRAGE*'))
print('mne_organize_dicom output fol: {}'.format(clin_full_fol))
if len(memprage_fols) > 0:
ret = au.is_true(
input(
'It seems like you already have memprage folders, are you sure you want to rerun?'
))
if not ret:
continue
utils.delete_folder_files(clin_full_fol)
fs_dir = utils.make_dir(op.join(args.fs_root, subject))
print('FreeSurfer output fol: {}'.format(fs_dir))
dicoms_full_path = op.join(args.dicoms_root, dicoms_fol)
if not op.isdir(dicoms_full_path):
print('{} does not exist!'.format(dicoms_full_path))
continue
rs = utils.partial_run_script(locals(), print_only=args.print_only)
os.chdir(clin_full_fol)
rs('mne_organize_dicom {dicoms_full_path}')
anat.recon_all(subject,
clin_full_fol,
overwrite=True,
subjects_dir=args.fs_root,
print_only=False,
n_jobs=args.n_jobs)
args = anat.read_cmd_args(
dict(
subject=subject,
function='all,create_skull_surfaces',
remote_subject_dir=op.join(args.fs_root, subject),
n_jobs=args.n_jobs,
))
pu.run_on_subjects(args, anat.main)
def run_on_subjects(args, main_func, subjects_itr=None, subject_func=None):
if subjects_itr is None:
subjects_itr = args.subject
subjects_flags, subjects_errors = {}, {}
args = init_args(args)
for tup in subjects_itr:
subject = get_subject(tup, subject_func)
utils.make_dir(op.join(MMVT_DIR, subject, 'mmvt'))
remote_subject_dir = utils.build_remote_subject_dir(
args.remote_subject_dir, subject)
logging.info(args)
print(
'****************************************************************')
print('subject: {}, atlas: {}'.format(subject, args.atlas))
print('remote dir: {}'.format(remote_subject_dir))
print(
'****************************************************************')
os.environ['SUBJECT'] = subject
flags = dict()
try:
# if utils.should_run(args, 'prepare_subject_folder'):
# I think we always want to run this
# *) Prepare the local subject's folder
flags['prepare_subject_folder'] = prepare_subject_folder(
subject, remote_subject_dir, args)
if not flags['prepare_subject_folder'] and not args.ignore_missing:
ans = input('Do you wish to continue (y/n)? ')
if not au.is_true(ans):
continue
flags['prepare_subject_folder'] = True
flags = main_func(tup, remote_subject_dir, args, flags)
subjects_flags[subject] = flags
except:
subjects_errors[subject] = traceback.format_exc()
print('Error in subject {}'.format(subject))
print(traceback.format_exc())
errors = defaultdict(list)
ret = True
good_subjects, bad_subjects = [], []
logs_fol = utils.make_dir(op.join(MMVT_DIR, subject, 'logs'))
logging.basicConfig(filename=op.join(logs_fol, 'preproc.log'),
level=logging.DEBUG)
for subject, flags in subjects_flags.items():
print('subject {}:'.format(subject))
logging.info('subject {}:'.format(subject))
for flag_type, val in flags.items():
print('{}: {}'.format(flag_type, val))
logging.info('{}: {}'.format(flag_type, val))
if not val:
errors[subject].append(flag_type)
if len(errors) > 0:
ret = False
print('Errors:')
logging.info('Errors:')
for subject, error in errors.items():
print('{}: {}'.format(subject, error))
logging.info('{}: {}'.format(subject, error))
for subject in subjects_flags.keys():
if len(errors[subject]) == 0:
good_subjects.append(subject)
else:
bad_subjects.append(subject)
print('Good subjects:\n {}'.format(good_subjects))
logging.info('Good subjects:\n {}'.format(good_subjects))
print('Bad subjects:\n {}'.format(bad_subjects))
logging.info('Good subjects:\n {}'.format(good_subjects))
utils.write_list_to_file(
good_subjects, op.join(utils.get_logs_fol(), 'good_subjects.txt'))
utils.write_list_to_file(bad_subjects,
op.join(utils.get_logs_fol(), 'bad_subjects.txt'))
return ret
def language(args):
# -f language -s nmr01361 --clinical_dir clin_4090354
# -s nmr01353 -f clean_4d_data --fsd sycabs --remote_fmri_dir "/space/megraid/clinical/MEG-MRI/seder/freesurfer" --nconditions 4
if args.clinical_dir == '':
print('You should set the clinical_dir first. Example: clin_4090354')
return
clinical_root_dir = op.join(args.remote_fmri_dir, args.clinical_dir)
if not op.isdir(clinical_root_dir):
print('{} does not exist!'.format(clinical_root_dir))
task = 'sycabs'
fwhm = 6
subject = args.subject[0]
remote_mri_dir = args.remote_clinical_subjects_dir
subject_mri_dir = op.join(remote_mri_dir, subject)
mri_subject_task_dir = utils.make_dir(op.join(subject_mri_dir, task))
clinical_dirs = glob.glob(op.join(clinical_root_dir, '*'))
clinical_dirs = [
d for d in clinical_dirs if utils.namebase(d) != 'mne_dcm'
]
remote_fmri_dir = utils.select_one_file(clinical_dirs)
fmri_fols = sorted(glob.glob(op.join(remote_fmri_dir, '*_SyCAbs')))
par_fol = utils.make_dir(op.join(remote_mri_dir, subject, 'par'))
par_files = glob.glob(op.join(par_fol, '*.par'))
sessions = sorted(
[utils.find_num_in_str(utils.namebase(d))[0] for d in fmri_fols])
# Warning: You first need to put the original ones in the following folder:
if len(par_files) == 0:
print('\n *** Please put the original par files in {} and rerun ***'.
format(op.join(remote_mri_dir, subject, 'par')))
return
par_files.sort(key=lambda x: int(utils.namebase(x).split('_')[-1]))
ret = input('''
Patient: {}
MRI folder: {}
fMRI root folder: {}
fMRI sessions: {}
Session and pars: {}
Do you want to continue (y/n)? '''.format(
subject, subject_mri_dir, remote_fmri_dir,
[utils.namebase(d) for d in fmri_fols],
list(zip([utils.namebase(f) for f in par_files], sessions))))
if not au.is_true(ret):
return
# You need first to run src.preproc.anatomy
if not op.isfile(anat.get_blend_fname(subject, args.atlas)):
args = anat.read_cmd_args(
dict(
subject=subject,
remote_subject_dir=subject_mri_dir,
ignore_missing=True,
))
pu.run_on_subjects(args, anat.main)
# convert the fMRI dicom files to nii
for fmri_fol in fmri_fols:
ses_num = utils.find_num_in_str(utils.namebase(fmri_fol))[0]
ses_files = glob.glob(op.join(fmri_fol, '**', '*.*'), recursive=True)
output_fname = op.join(
utils.make_dir(op.join(mri_subject_task_dir, ses_num)), 'f.nii.gz')
if not op.isfile(output_fname):
fu.mri_convert(ses_files[0], output_fname)
# Convert and arrange the par file
from src.misc.fmri_scripts import convert_par
for par_file, session in zip(par_files, sessions):
fs_par_fname = op.join(mri_subject_task_dir, session,
'{}.par'.format(task))
# if not op.isfile(fs_par_fname):
warnings = convert_par.sycabs(par_file, fs_par_fname)
if warnings != '':
print(
'\n *** Please fix the problems with the par convertion ({}) and rerun ***\n'
.format(par_file))
return
for hemi in utils.HEMIS:
utils.delete_folder_files(
op.join(remote_mri_dir, '{}_sm{}_{}'.format(task, fwhm, hemi)))
# Run the FreeSurfer analysis
args = fmri.read_cmd_args(
dict(subject=subject,
atlas=args.atlas,
function='clean_4d_data',
fsd=task,
fwhm=fwhm,
remote_fmri_dir=remote_mri_dir,
nconditions=4,
ignore_missing=True,
print_only=False,
overwrite_4d_preproc=False))
pu.run_on_subjects(args, fmri.main)
# Load the fMRI results
args = fmri.read_cmd_args(
dict(
subject=subject,
atlas=args.atlas,
function='load_surf_files',
fmri_file_template=op.join(MMVT_DIR, subject, 'fmri',
'words_v_symbols_{hemi}.mgz'),
))
pu.run_on_subjects(args, fmri.main)