Ejemplo n.º 1
0
def run_function_in_parallel(func,
                             all_params,
                             n_jobs,
                             split_jobs=True,
                             verbose=True):
    # Filter our items that should not run (all input files should exist, and all output files shoud not)
    params = [
        p for p in all_params
        if all(_run_func_in_parallel((func, [p], True, 0, verbose)))
    ]
    print('*** Run {}/{} records, {} jobs ***'.format(len(params),
                                                      len(all_params), n_jobs))
    ret = input('ok? (y/n) ')
    if not au.is_true(ret):
        return
    if split_jobs:
        chunks_indices = np.array_split(np.arange(len(params)), n_jobs)
        chunk_params = [
            (func, [params[ind]
                    for ind in chunk_indices], False, thread_ind, verbose)
            for thread_ind, chunk_indices in enumerate(chunks_indices)
        ]
    else:
        chunk_params = [(func, [p], False, thread_ind, verbose)
                        for thread_ind, p in enumerate(params)]
    utils.run_parallel(_run_func_in_parallel, chunk_params, n_jobs)
Ejemplo n.º 2
0
Archivo: ct.py Proyecto: ltirrell/mmvt
def convert_ct_to_mgz(subject,
                      ct_raw_input_fol,
                      ct_fol='',
                      output_name='ct_org.mgz',
                      overwrite=False,
                      print_only=False,
                      ask_before=False):
    if not op.isdir(ct_fol):
        ct_fol = utils.make_dir(op.join(MMVT_DIR, subject, 'ct'))
    if op.isfile(op.join(ct_fol, 'ct_reg_to_mr.mgz')) and not overwrite:
        return True
    if op.isfile(op.join(ct_fol, output_name)):
        return True
    ct_fname = utils.select_one_file(glob.glob(op.join(ct_fol, '*.mgz')))
    if op.isfile(ct_fname):
        if utils.namebase(ct_fname) != 'ct_org':
            utils.make_link(ct_fname, op.join(ct_fol, 'ct_org.mgz'))
        return True
    output_fname = op.join(ct_fol, output_name)
    if op.isfile(output_fname):
        if not overwrite:
            return True
        else:
            os.remove(output_fname)
    if op.isfile(op.join(SUBJECTS_DIR, subject, 'ct', 'ct.nii.gz')):
        ct_files = [op.join(SUBJECTS_DIR, subject, 'ct', 'ct.nii.gz')]
    elif op.isfile(op.join(SUBJECTS_DIR, subject, 'mri', 'ct.nii.gz')):
        ct_files = [op.join(SUBJECTS_DIR, subject, 'mri', 'ct.nii.gz')]
    else:
        if not op.isdir(ct_raw_input_fol):
            print('{} does not exist!'.format(ct_fol))
            return False
        ct_files = glob.glob(op.join(ct_raw_input_fol, '*.dcm'))
        if len(ct_files) == 0:
            sub_folders = [
                d for d in glob.glob(op.join(ct_raw_input_fol, '*'))
                if op.isdir(d)
            ]
            if len(sub_folders) == 0:
                print('Cannot find CT files in {}!'.format(ct_raw_input_fol))
                return False
            fol = utils.select_one_file(sub_folders, '', 'CT', is_dir=True)
            ct_files = glob.glob(op.join(fol, '*.dcm'))
            if len(ct_files) == 0:
                print('Cannot find CT files in {}!'.format(fol))
                return False
        ct_files.sort(key=op.getmtime)
        if ask_before:
            ret = input('convert {} to {}? '.format(ct_files[0], output_fname))
            if not au.is_true(ret):
                return False
    fu.mri_convert(ct_files[0], output_fname, print_only=print_only)
    return True if print_only else op.isfile(output_fname)
Ejemplo n.º 3
0
def run_on_subjects(args, main_func, subjects_itr=None, subject_func=None):
    if subjects_itr is None:
        subjects_itr = args.subject
    subjects_flags, subjects_errors = {}, {}
    args.n_jobs = utils.get_n_jobs(args.n_jobs)
    args.sftp_password = utils.get_sftp_password(
        args.subject, SUBJECTS_DIR, args.necessary_files, args.sftp_username, args.overwrite_fs_files) \
        if args.sftp else ''
    if '*' in args.subject:
        args.subject = [utils.namebase(fol) for fol in glob.glob(op.join(SUBJECTS_DIR, args.subject))]
    os.environ['SUBJECTS_DIR'] = SUBJECTS_DIR
    for tup in subjects_itr:
        subject = get_subject(tup, subject_func)
        utils.make_dir(op.join(MMVT_DIR, subject, 'mmvt'))
        remote_subject_dir = utils.build_remote_subject_dir(args.remote_subject_dir, subject)
        print('****************************************************************')
        print('subject: {}, atlas: {}'.format(subject, args.atlas))
        print('remote dir: {}'.format(remote_subject_dir))
        print('****************************************************************')
        os.environ['SUBJECT'] = subject
        flags = dict()
        if args.necessary_files == '':
            args.necessary_files = dict()
        try:
            if utils.should_run(args, 'prepare_local_subjects_folder'):
                # *) Prepare the local subject's folder
                flags['prepare_local_subjects_folder'] = prepare_local_subjects_folder(
                    subject, remote_subject_dir, args)
                if not flags['prepare_local_subjects_folder'] and not args.ignore_missing:
                    ans = input('Do you which to continue (y/n)? ')
                    if not au.is_true(ans):
                        continue

            flags = main_func(tup, remote_subject_dir, args, flags)
            subjects_flags[subject] = flags
        except:
            subjects_errors[subject] = traceback.format_exc()
            print('Error in subject {}'.format(subject))
            print(traceback.format_exc())

    errors = defaultdict(list)
    for subject, flags in subjects_flags.items():
        print('subject {}:'.format(subject))
        for flag_type, val in flags.items():
            print('{}: {}'.format(flag_type, val))
            if not val:
                errors[subject].append(flag_type)
    if len(errors) > 0:
        print('Errors:')
        for subject, error in errors.items():
            print('{}: {}'.format(subject, error))
Ejemplo n.º 4
0
def recon_all_clin(args):
    # python -m src.preproc.examples.anatomy -s nmr01426 -f recon_all_clin --clin_fol clin_6966926 --dicoms_fol Prisma_fit-67026-20200618-141203-000586
    import os
    for subject, clin_fol, dicoms_fol in zip(args.subject, args.clin_fol,
                                             args.dicoms_fol):
        clin_full_fol = utils.make_dir(
            op.join(args.clin_root, clin_fol, 'mne_dicom'))
        memprage_fols = glob.glob(op.join(clin_full_fol, '*MEMPRAGE*'))
        print('mne_organize_dicom output fol: {}'.format(clin_full_fol))
        if len(memprage_fols) > 0:
            ret = au.is_true(
                input(
                    'It seems like you already have memprage folders, are you sure you want to rerun?'
                ))
            if not ret:
                continue
            utils.delete_folder_files(clin_full_fol)
        fs_dir = utils.make_dir(op.join(args.fs_root, subject))
        print('FreeSurfer output fol: {}'.format(fs_dir))
        dicoms_full_path = op.join(args.dicoms_root, dicoms_fol)
        if not op.isdir(dicoms_full_path):
            print('{} does not exist!'.format(dicoms_full_path))
            continue
        rs = utils.partial_run_script(locals(), print_only=args.print_only)
        os.chdir(clin_full_fol)
        rs('mne_organize_dicom {dicoms_full_path}')
        anat.recon_all(subject,
                       clin_full_fol,
                       overwrite=True,
                       subjects_dir=args.fs_root,
                       print_only=False,
                       n_jobs=args.n_jobs)
        args = anat.read_cmd_args(
            dict(
                subject=subject,
                function='all,create_skull_surfaces',
                remote_subject_dir=op.join(args.fs_root, subject),
                n_jobs=args.n_jobs,
            ))
        pu.run_on_subjects(args, anat.main)
Ejemplo n.º 5
0
def run_on_subjects(args, main_func, subjects_itr=None, subject_func=None):
    if subjects_itr is None:
        subjects_itr = args.subject
    subjects_flags, subjects_errors = {}, {}
    args = init_args(args)
    for tup in subjects_itr:
        subject = get_subject(tup, subject_func)
        utils.make_dir(op.join(MMVT_DIR, subject, 'mmvt'))
        remote_subject_dir = utils.build_remote_subject_dir(
            args.remote_subject_dir, subject)
        logging.info(args)
        print(
            '****************************************************************')
        print('subject: {}, atlas: {}'.format(subject, args.atlas))
        print('remote dir: {}'.format(remote_subject_dir))
        print(
            '****************************************************************')
        os.environ['SUBJECT'] = subject
        flags = dict()
        try:
            # if utils.should_run(args, 'prepare_subject_folder'):
            # I think we always want to run this
            # *) Prepare the local subject's folder
            flags['prepare_subject_folder'] = prepare_subject_folder(
                subject, remote_subject_dir, args)
            if not flags['prepare_subject_folder'] and not args.ignore_missing:
                ans = input('Do you wish to continue (y/n)? ')
                if not au.is_true(ans):
                    continue
            flags['prepare_subject_folder'] = True

            flags = main_func(tup, remote_subject_dir, args, flags)
            subjects_flags[subject] = flags
        except:
            subjects_errors[subject] = traceback.format_exc()
            print('Error in subject {}'.format(subject))
            print(traceback.format_exc())

    errors = defaultdict(list)
    ret = True
    good_subjects, bad_subjects = [], []
    logs_fol = utils.make_dir(op.join(MMVT_DIR, subject, 'logs'))
    logging.basicConfig(filename=op.join(logs_fol, 'preproc.log'),
                        level=logging.DEBUG)
    for subject, flags in subjects_flags.items():
        print('subject {}:'.format(subject))
        logging.info('subject {}:'.format(subject))
        for flag_type, val in flags.items():
            print('{}: {}'.format(flag_type, val))
            logging.info('{}: {}'.format(flag_type, val))
            if not val:
                errors[subject].append(flag_type)
    if len(errors) > 0:
        ret = False
        print('Errors:')
        logging.info('Errors:')
        for subject, error in errors.items():
            print('{}: {}'.format(subject, error))
            logging.info('{}: {}'.format(subject, error))
    for subject in subjects_flags.keys():
        if len(errors[subject]) == 0:
            good_subjects.append(subject)
        else:
            bad_subjects.append(subject)
    print('Good subjects:\n {}'.format(good_subjects))
    logging.info('Good subjects:\n {}'.format(good_subjects))
    print('Bad subjects:\n {}'.format(bad_subjects))
    logging.info('Good subjects:\n {}'.format(good_subjects))
    utils.write_list_to_file(
        good_subjects, op.join(utils.get_logs_fol(), 'good_subjects.txt'))
    utils.write_list_to_file(bad_subjects,
                             op.join(utils.get_logs_fol(), 'bad_subjects.txt'))
    return ret
Ejemplo n.º 6
0
def language(args):
    # -f language -s nmr01361 --clinical_dir clin_4090354
    # -s nmr01353 -f clean_4d_data --fsd sycabs --remote_fmri_dir "/space/megraid/clinical/MEG-MRI/seder/freesurfer" --nconditions 4
    if args.clinical_dir == '':
        print('You should set the clinical_dir first. Example: clin_4090354')
        return
    clinical_root_dir = op.join(args.remote_fmri_dir, args.clinical_dir)
    if not op.isdir(clinical_root_dir):
        print('{} does not exist!'.format(clinical_root_dir))

    task = 'sycabs'
    fwhm = 6
    subject = args.subject[0]
    remote_mri_dir = args.remote_clinical_subjects_dir
    subject_mri_dir = op.join(remote_mri_dir, subject)
    mri_subject_task_dir = utils.make_dir(op.join(subject_mri_dir, task))
    clinical_dirs = glob.glob(op.join(clinical_root_dir, '*'))
    clinical_dirs = [
        d for d in clinical_dirs if utils.namebase(d) != 'mne_dcm'
    ]
    remote_fmri_dir = utils.select_one_file(clinical_dirs)
    fmri_fols = sorted(glob.glob(op.join(remote_fmri_dir, '*_SyCAbs')))
    par_fol = utils.make_dir(op.join(remote_mri_dir, subject, 'par'))
    par_files = glob.glob(op.join(par_fol, '*.par'))
    sessions = sorted(
        [utils.find_num_in_str(utils.namebase(d))[0] for d in fmri_fols])

    # Warning: You first need to put the original ones in the following folder:
    if len(par_files) == 0:
        print('\n *** Please put the original par files in {} and rerun ***'.
              format(op.join(remote_mri_dir, subject, 'par')))
        return

    par_files.sort(key=lambda x: int(utils.namebase(x).split('_')[-1]))
    ret = input('''
        Patient: {}
        MRI folder: {}
        fMRI root folder: {}
        fMRI sessions: {}
        Session and pars: {}
        Do you want to continue (y/n)? '''.format(
        subject, subject_mri_dir, remote_fmri_dir,
        [utils.namebase(d) for d in fmri_fols],
        list(zip([utils.namebase(f) for f in par_files], sessions))))
    if not au.is_true(ret):
        return

    # You need first to run src.preproc.anatomy
    if not op.isfile(anat.get_blend_fname(subject, args.atlas)):
        args = anat.read_cmd_args(
            dict(
                subject=subject,
                remote_subject_dir=subject_mri_dir,
                ignore_missing=True,
            ))
        pu.run_on_subjects(args, anat.main)

    # convert the fMRI dicom files to nii
    for fmri_fol in fmri_fols:
        ses_num = utils.find_num_in_str(utils.namebase(fmri_fol))[0]
        ses_files = glob.glob(op.join(fmri_fol, '**', '*.*'), recursive=True)
        output_fname = op.join(
            utils.make_dir(op.join(mri_subject_task_dir, ses_num)), 'f.nii.gz')
        if not op.isfile(output_fname):
            fu.mri_convert(ses_files[0], output_fname)

    # Convert and arrange the par file
    from src.misc.fmri_scripts import convert_par
    for par_file, session in zip(par_files, sessions):
        fs_par_fname = op.join(mri_subject_task_dir, session,
                               '{}.par'.format(task))
        # if not op.isfile(fs_par_fname):
        warnings = convert_par.sycabs(par_file, fs_par_fname)
        if warnings != '':
            print(
                '\n *** Please fix the problems with the par convertion ({}) and rerun ***\n'
                .format(par_file))
            return

    for hemi in utils.HEMIS:
        utils.delete_folder_files(
            op.join(remote_mri_dir, '{}_sm{}_{}'.format(task, fwhm, hemi)))

    # Run the FreeSurfer analysis
    args = fmri.read_cmd_args(
        dict(subject=subject,
             atlas=args.atlas,
             function='clean_4d_data',
             fsd=task,
             fwhm=fwhm,
             remote_fmri_dir=remote_mri_dir,
             nconditions=4,
             ignore_missing=True,
             print_only=False,
             overwrite_4d_preproc=False))
    pu.run_on_subjects(args, fmri.main)

    # Load the fMRI results
    args = fmri.read_cmd_args(
        dict(
            subject=subject,
            atlas=args.atlas,
            function='load_surf_files',
            fmri_file_template=op.join(MMVT_DIR, subject, 'fmri',
                                       'words_v_symbols_{hemi}.mgz'),
        ))
    pu.run_on_subjects(args, fmri.main)