def submit_job(cmd, i, walltime="23:00:00"):
if DRYRUN:
return
job_name = 'dm_freesurfer_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
# Bit of an ugly hack to allow job submission on the scc. Should be replaced
# with drmaa or some other queue interface later
if SYSTEM is 'kimel':
job_file = '/tmp/{}'.format(job_name)
with open(job_file, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
job = "qsub -V -q main.q -N {} {}".format(job_name, job_file)
rtn, out = utils.run(job)
else:
job = "echo {} | qbatch -N {} --logdir {} --walltime {} -".format(
cmd, job_name, LOG_DIR, walltime)
rtn, out = utils.run(job, specialquote=False)
if rtn:
logger.error("Job submission failed.")
if out:
logger.error("stdout: {}".format(out))
sys.exit(1)
def update_aggregate_stats(config):
logger.info("Updating aggregate stats")
freesurfer_dir = config.get_path('freesurfer')
enigma_ctx = os.path.join(config.system_config['DATMAN_ASSETSDIR'],
'ENGIMA_ExtractCortical.sh')
enigma_sub = os.path.join(config.system_config['DATMAN_ASSETSDIR'],
'ENGIMA_ExtractSubcortical.sh')
utils.run('{} {} {}'.format(enigma_ctx, freesurfer_dir,
config.study_config['STUDY_TAG']), dryrun=DRYRUN)
utils.run('{} {} {}'.format(enigma_sub, freesurfer_dir,
config.study_config['STUDY_TAG']), dryrun=DRYRUN)
def update_aggregate_stats(config):
logger.info("Updating aggregate stats")
freesurfer_dir = config.get_path('freesurfer')
enigma_ctx = os.path.join(config.system_config['DATMAN_ASSETSDIR'],
'ENGIMA_ExtractCortical.sh')
enigma_sub = os.path.join(config.system_config['DATMAN_ASSETSDIR'],
'ENGIMA_ExtractSubcortical.sh')
utils.run('{} {} {}'.format(enigma_ctx, freesurfer_dir,
config.study_config['STUDY_TAG']), dryrun=DRYRUN)
utils.run('{} {} {}'.format(enigma_sub, freesurfer_dir,
config.study_config['STUDY_TAG']), dryrun=DRYRUN)
def make_epitome_folders(path, n_runs):
"""
Makes an epitome-compatible folder structure with functional data FUNC of n
import pipesruns, and a single T1.
This works assuming we've run everything through freesurfer.
If we need multisession, it might make sense to run this multiple times
(once per session).
"""
utils.run('mkdir -p ' + path + '/TEMP/SUBJ/T1/SESS01/RUN01')
for r in np.arange(n_runs)+1:
num = "{:0>2}".format(str(r))
utils.run('mkdir -p ' + path + '/TEMP/SUBJ/FUNC/SESS01/RUN' + num)
def make_epitome_folders(path, n_runs):
"""
Makes an epitome-compatible folder structure with functional data FUNC of n
import pipesruns, and a single T1.
This works assuming we've run everything through freesurfer.
If we need multisession, it might make sense to run this multiple times
(once per session).
"""
utils.run('mkdir -p ' + path + '/TEMP/SUBJ/T1/SESS01/RUN01')
for r in np.arange(n_runs) + 1:
num = "{:0>2}".format(str(r))
utils.run('mkdir -p ' + path + '/TEMP/SUBJ/FUNC/SESS01/RUN' + num)
def run_hcp_convert(path, config, study):
"""Runs fs2hcp on the input subject"""
study_base = config.get_study_base(study)
subject = os.path.basename(path)
freesurfer_dir = os.path.join(study_base, config.site_config['paths']['freesurfer'])
hcp_dir = os.path.join(study_base, config.site_config['paths']['hcp'])
output_dir = utils.define_folder(os.path.join(hcp_dir, subject))
if outputs_exist(output_dir):
logger.debug('outputs found in {}'.format(path))
sys.exit()
# reset / remove error.log
error_log = os.path.join(output_dir, 'error.log')
if os.path.isfile(error_log):
os.remove(error_log)
# run fs2hcp
command = 'fs2hcp --FSpath={} --HCPpath={} --subject={}'.format(freesurfer_dir, hcp_dir, subject)
rtn, out = utils.run(command)
if rtn:
error_message = "fs2hcp failed: {}\n{}".format(command, out)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
def create_indices_bm(config, study):
hcp_dir = config.get_path('hcp')
command = 'cifti_vis_recon_all index --hcp-data-dir {}'.format(hcp_dir)
rtn, out = utils.run(command)
if rtn:
error_message = "fs2hcp failed: {}\n{}".format(command, out)
logger.debug(error_message)
def create_indices_bm(config, study):
hcp_dir = config.get_path('hcp')
if os.path.exists(os.path.join(hcp_dir, 'qc_recon_all')):
command = 'cifti_vis_recon_all index --hcp-data-dir {}'.format(hcp_dir)
rtn, out = utils.run(command)
if rtn:
error_message = "qc index creation failed: {}\n{}".format(command, out)
logger.debug(error_message)
else:
logger.debug('qc_recon_all directory does not exist, not generating index')
def run_pipeline(config, subject, t1, t2):
if not input_exists(t1) or not input_exists(t2):
sys.exit(1)
base_dir = utils.define_folder(config.get_path('hcp_fs'))
dest_dir = utils.define_folder(os.path.join(base_dir, subject))
with utils.cd(dest_dir):
hcp_pipeline = "hcp-freesurfer.sh {} {} {} {}".format(base_dir, subject,
t1, t2)
rtn, out = utils.run(hcp_pipeline, dryrun=DRYRUN)
if rtn:
logger.error("hcp-freesurfer.sh exited with non-zero status code. "
"Output: {}".format(out))
def run_hcp_convert(path, config, study):
"""Runs fs2hcp on the input subject"""
study_base = config.get_study_base(study)
subject = os.path.basename(path)
freesurfer_dir = config.get_path('freesurfer')
hcp_dir = config.get_path('hcp')
output_dir = os.path.join(hcp_dir, subject)
# run fs2hcp
#command = 'fs2hcp --FSpath={} --HCPpath={} --subject={}'.format(freesurfer_dir, hcp_dir, subject)
command = 'ciftify_recon_all --fs-subjects-dir {} --hcp-data-dir {} {}'.format(freesurfer_dir, hcp_dir, subject)
rtn, out = utils.run(command)
if rtn:
error_message = "ciftify_recon_all failed: {}\n{}".format(command, out)
logger.debug(error_message)
command2 = make_vis_cmd(hcp_dir, subject)
rtn, out = utils.run(command2)
if rtn:
error_message = "cifti_vis_recon_all snaps failed: {}\n{}".format(command2, out)
logger.debug(error_message)
def run_pipeline(config, subject, t1, t2):
if not input_exists(t1) or not input_exists(t2):
sys.exit(1)
base_dir = utils.define_folder(config.get_path('hcp_fs'))
dest_dir = utils.define_folder(os.path.join(base_dir, subject))
with utils.cd(dest_dir):
hcp_pipeline = "hcp-freesurfer.sh {} {} {} {}".format(
base_dir, subject, t1, t2)
rtn, out = utils.run(hcp_pipeline, dryrun=DRYRUN)
if rtn:
logger.error("hcp-freesurfer.sh exited with non-zero status code. "
"Output: {}".format(out))
def submit_job(cmd, subid, log_dir, walltime="36:00:00"):
job_name = "dm_hcp_freesurfer_{}_{}".format(subid,
time.strftime("%Y%m%d-%H%M%S"))
rtn, out = utils.run("echo {} | qbatch -N {} --walltime {} "
"--logdir {} -".format(cmd, job_name, walltime, log_dir),
specialquote=False, dryrun=DRYRUN)
if rtn:
logger.error("Job submission failed.")
if out:
logger.error("stdout: {}".format(out))
sys.exit(1)
def run_hcp_convert(path, config, study):
"""Runs fs2hcp on the input subject"""
study_base = config.get_study_base(study)
subject = os.path.basename(path)
freesurfer_dir = config.get_path('freesurfer')
hcp_dir = config.get_path('hcp')
output_dir = os.path.join(hcp_dir, subject)
# run fs2hcp
#command = 'fs2hcp --FSpath={} --HCPpath={} --subject={}'.format(freesurfer_dir, hcp_dir, subject)
command = 'ciftify_recon_all --fs-subjects-dir {} --hcp-data-dir {} {}'.format(
freesurfer_dir, hcp_dir, subject)
rtn, out = utils.run(command)
if rtn:
error_message = "fs2hcp failed: {}\n{}".format(command, out)
logger.debug(error_message)
command2 = 'cifti_vis_recon_all snaps --hcp-data-dir {} {}'.format(
hcp_dir, subject)
rtn, out = utils.run(command2)
if rtn:
error_message = "fs2hcp failed: {}\n{}".format(command2, out)
logger.debug(error_message)
def submit_job(cmd, i, walltime="24:00:00"):
if DRYRUN:
return
job_name = 'dm_freesurfer_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
job_file = '/tmp/{}'.format(job_name)
with open(job_file, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run("qsub -V -q main.q -N {} {}".format(
job_name, job_file))
if rtn:
logger.error("Job submission failed.")
if out:
logger.error("stdout: {}".format(out))
sys.exit(1)
def run_all(nrrd_dir, config, study):
"""Finds all non-phantom input nrrds and run unring.py in serial."""
study_base = config.get_study_base(study)
subjects = os.listdir(nrrd_dir)
subjects = filter(lambda x: '_PHA_' not in x, subjects)
unring_dir = utils.define_folder(
os.path.join(study_base, config.get_path('unring')))
tags = config.study_config['unring']['tags']
for subject in subjects:
output_dir = utils.define_folder(os.path.join(unring_dir, subject))
inputs = os.listdir(os.path.join(nrrd_dir, subject))
inputs = select_inputs(inputs,
tags) # selects inputs with matching tag
for input_file in inputs:
# don't run if the outputs of unring already exist
if outputs_exist(output_dir, input_file):
continue
# reset / remove error.log
error_log = os.path.join(output_dir, 'error.log')
if os.path.isfile(error_log):
os.remove(error_log)
output_fname = os.path.join(output_dir, input_file)
input_fname = os.path.join(nrrd_dir, subject, input_file)
cmd = 'unring.py {} {} -v'.format(input_fname, output_fname)
logger.debug('running {}'.format(cmd))
rtn, out = utils.run(cmd)
if rtn:
error_message = "unring.py failed: {}\n{}".format(cmd, out)
logger.info(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
else:
pass
def run_freesurfer(subject, blacklist, config, resubmit=False):
"""Finds the inputs for subject and runs freesurfer."""
freesurfer_path = config.get_path('freesurfer')
output_dir = os.path.join(freesurfer_path, subject.full_id)
if outputs_exist(output_dir) and not resubmit:
return
# reset / remove error.log
error_log = os.path.join(LOG_DIR, '{}_error.log'.format(subject.full_id))
if os.path.isfile(error_log):
os.remove(error_log)
args = get_freesurfer_arguments(config, subject.site)
scripts_dir = os.path.join(output_dir, 'scripts')
if outputs_exist(output_dir):
# If outputs exist and the script didnt return above, it means
# 'resubmit' == True and the subject must be restarted
remove_IsRunning(scripts_dir)
input_files = []
else:
input_files = get_anatomical_images('tags', 'i', subject, blacklist, config,
error_log)
optional_files = get_optional_images(subject, blacklist, config,
error_log)
input_files.extend(optional_files)
command = "recon-all {args} -subjid {subid} {inputs}".format(args=args,
subid=subject.full_id, inputs=" ".join(input_files))
logger.info('Running recon-all')
rtn, out = utils.run(command, dryrun=DRYRUN)
if rtn:
error_message = 'freesurfer failed: {}\n{}'.format(command, out)
logger.debug(error_message)
write_lines(error_log, '{}\n{}'.format(error_message, NODE))
def run_freesurfer(subject, blacklist, config, resubmit=False):
"""Finds the inputs for subject and runs freesurfer."""
freesurfer_path = config.get_path('freesurfer')
output_dir = os.path.join(freesurfer_path, subject.full_id)
if outputs_exist(output_dir) and not resubmit:
return
# reset / remove error.log
error_log = os.path.join(LOG_DIR, '{}_error.log'.format(subject.full_id))
if os.path.isfile(error_log):
os.remove(error_log)
args = get_freesurfer_arguments(config, subject.site)
scripts_dir = os.path.join(output_dir, 'scripts')
if outputs_exist(output_dir):
# If outputs exist and the script didnt return above, it means
# 'resubmit' == True and the subject must be restarted
remove_IsRunning(scripts_dir)
input_files = []
else:
input_files = get_anatomical_images('tags', 'i', subject, blacklist,
config, error_log)
optional_files = get_optional_images(subject, blacklist, config,
error_log)
input_files.extend(optional_files)
command = "recon-all {args} -subjid {subid} {inputs}".format(
args=args, subid=subject.full_id, inputs=" ".join(input_files))
logger.info('Running recon-all')
rtn, out = utils.run(command, dryrun=DRYRUN)
if rtn:
error_message = 'freesurfer failed: {}\n{}'.format(command, out)
logger.debug(error_message)
write_lines(error_log, '{}\n{}'.format(error_message, NODE))
def run_all(nrrd_dir, config, study):
"""Finds all non-phantom input nrrds and run unring.py in serial."""
study_base = config.get_study_base(study)
subjects = os.listdir(nrrd_dir)
subjects = filter(lambda x: '_PHA_' not in x, subjects)
unring_dir = utils.define_folder(os.path.join(study_base, config.get_path('unring')))
tags = config.study_config['unring']['tags']
for subject in subjects:
output_dir = utils.define_folder(os.path.join(unring_dir, subject))
inputs = os.listdir(os.path.join(nrrd_dir, subject))
inputs = select_inputs(inputs, tags) # selects inputs with matching tag
for input_file in inputs:
# don't run if the outputs of unring already exist
if outputs_exist(output_dir, input_file):
continue
# reset / remove error.log
error_log = os.path.join(output_dir, 'error.log')
if os.path.isfile(error_log):
os.remove(error_log)
output_fname = os.path.join(output_dir, input_file)
input_fname = os.path.join(nrrd_dir, subject, input_file)
cmd = 'unring.py {} {} -v'.format(input_fname, output_fname)
logger.debug('running {}'.format(cmd))
rtn, out = utils.run(cmd)
if rtn:
error_message = "unring.py failed: {}\n{}".format(cmd, out)
logger.info(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
else:
pass
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['freesurfer', 'hcp']:
if k not in config.site_config['paths']:
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
freesurfer_dir = config.get_path('freesurfer')
hcp_dir = config.get_path('hcp')
if scanid:
path = os.path.join(freesurfer_dir, scanid)
try:
run_hcp_convert(path, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
return
config = cfg.config(study=study)
qced_subjects = config.get_subject_metadata()
# running for batch mode
new_subjects = []
# find subjects where at least one expected output does not exist
for subject in qced_subjects:
subj_dir = os.path.join(hcp_dir, subject)
if not ciftify_outputs_exist(subj_dir):
if fs_outputs_exist(os.path.join(freesurfer_dir, subject)):
new_subjects.append(subject)
create_indices_bm(config, study)
# submit a list of calls to ourself, one per subject
commands = []
if debug:
debugopt = '--debug'
else:
debugopt = ''
for subject in new_subjects:
commands.append(" ".join(
[__file__, study, '--subject {} '.format(subject), debugopt]))
if commands:
logger.debug('queueing up the following commands:\n' +
'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_fs2hcp_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def main():
"""
Loops through subjects, preprocessing using supplied script, and runs a
first-level GLM using AFNI (tent functions, 15 s window) on all subjects.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
subject = arguments['--subject']
debug = arguments['--debug']
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
imob_dir = os.path.join(study_base, config.get_path('fmri'), 'imob')
# process a single subject
if subject:
# get required inputs from each
files = glob.glob(os.path.join(imob_dir, subject) + '/*.nii.gz')
inputs = get_inputs(files, config)
# check if subject has already been processed
if check_complete(imob_dir, subject):
logger.info('{} already analysed'.format(subject))
sys.exit(0)
# first level GLM for inputs
for input_type in inputs.keys():
script = generate_analysis_script(subject, inputs, input_type, config, study)
rtn, out = utils.run('chmod 754 {}'.format(script))
rtn, out = utils.run(script)
if rtn:
logger.error('Script {} failed to run on subject {} with error:\n{}'.format(
script, subject, out))
sys.exit(1)
# process all subjects
else:
commands = []
for path in glob.glob('{}/*'.format(imob_dir)):
subject = os.path.basename(path)
# add subject if any of the expected outputs do not exist
files = glob.glob(os.path.join(imob_dir, subject) + '/*.nii.gz')
try:
inputs = get_inputs(files, config)
except:
logger.debug('Invalid inputs for {}'.format(subject))
continue
expected = inputs.keys()
for exp in expected:
if not filter(lambda x: '{}_glm_IM_1stlvl_{}'.format(subject, exp) in x, files):
commands.append(" ".join([__file__, study, '--subject {}'.format(subject)]))
break
if commands:
logger.debug("queueing up the following commands:\n"+'\n'.join(commands))
#fd, path = tempfile.mkstemp()
#os.write(fd, '\n'.join(commands))
#os.close(fd)
for i, cmd in enumerate(commands):
jobname = "dm_imob_{}_{}".format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
#rtn, out, err = utils.run('qbatch -i --logdir {logdir} -N {name} --walltime {wt} {cmds}'.format(logdir = log_path, name = jobname, wt = walltime, cmds = path))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
logger.error(msg)
finally:
f1.close()
f2.close()
f3.close()
f4.close()
# run the GLM
files = glob.glob(os.path.join(ea_dir, subject + '/*.nii.gz'))
inputs = get_inputs(files, config)
for input_type in inputs.keys():
script = generate_analysis_script(subject, inputs, input_type, config,
study)
rtn, out = utils.run('chmod 754 {}'.format(script))
rtn, out = utils.run(script)
if rtn:
logger.error(
'Script {} failed to run on subject {} with error:\n{}'.format(
script, subject, out))
sys.exit(1)
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
subject = arguments['--subject']
debug = arguments['--debug']
def main():
"""
Runs fmri data through the specified epitome script.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['nii', 'fmri', 'hcp']:
if k not in config.site_config['paths']:
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
for x in config.study_config['fmri'].iteritems():
for k in ['dims', 'del', 'pipeline', 'tags', 'export', 'tr']:
if k not in x[1].keys():
logger.error("fmri:{}:{} not defined in configuration file".format(x[0], k))
sys.exit(1)
nii_dir = os.path.join(study_base, config.site_config['paths']['nii'])
if scanid:
path = os.path.join(nii_dir, scanid)
if '_PHA_' in scanid:
sys.exit('Subject {} if a phantom, cannot be analyzed'.format(scanid))
try:
run_epitome(path, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
# run in batch mode
else:
subjects = []
nii_dirs = glob.glob('{}/*'.format(nii_dir))
# find subjects where at least one expected output does not exist
for path in nii_dirs:
subject = os.path.basename(path)
if sid.is_phantom(subject):
logger.debug("Subject {} is a phantom. Skipping.".format(subject))
continue
fmri_dir = utils.define_folder(os.path.join(study_base, config.site_config['paths']['fmri']))
for exp in config.study_config['fmri'].keys():
expected_names = config.study_config['fmri'][exp]['export']
subj_dir = os.path.join(fmri_dir, exp, subject)
if not outputs_exist(subj_dir, expected_names):
subjects.append(subject)
break
subjects = list(set(subjects))
# submit a list of calls to ourself, one per subject
commands = []
if debug:
debugopt = '--debug'
else:
debugopt = ''
for subject in subjects:
commands.append(" ".join(['python ', __file__, study, '--subject {} '.format(subject), debugopt]))
if commands:
logger.debug('queueing up the following commands:\n'+'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_fmri_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def run_epitome(path, config, study):
"""
Finds the appropriate inputs for input subject, builds a temporary epitome
folder, runs epitome, and finally copies the outputs to the fmri_dir.
"""
study_base = config.get_study_base(study)
subject = os.path.basename(path)
nii_dir = os.path.join(study_base, config.site_config['paths']['nii'])
t1_dir = os.path.join(study_base, config.site_config['paths']['hcp'])
fmri_dir = utils.define_folder(os.path.join(study_base, config.site_config['paths']['fmri']))
experiments = config.study_config['fmri'].keys()
# run file collection --> epitome --> export for each study
logger.debug('experiments found {}'.format(experiments))
for exp in experiments:
logger.debug('running experiment {}'.format(exp))
# collect the files needed for each experiment
expected_names = config.study_config['fmri'][exp]['export']
expected_tags = config.study_config['fmri'][exp]['tags']
output_dir = utils.define_folder(os.path.join(fmri_dir, exp, subject))
# don't run if the outputs of epitome already exist
if outputs_exist(output_dir, expected_names):
continue
# reset / remove error.log
error_log = os.path.join(output_dir, 'error.log')
if os.path.isfile(error_log):
os.remove(error_log)
failed = False
if type(expected_tags) == str:
expected_tags = [expected_tags]
# locate functional data
files = glob.glob(path + '/*')
functionals = []
for tag in expected_tags:
candidates = filter(lambda x: tag in x, files)
candidates = utils.filter_niftis(candidates)
candidates.sort()
logger.debug('checking functional inputs {}'.format(candidates))
try:
check_inputs(config, tag, path, candidates)
except Exception as m:
error_message = 'Did not find the correct number of fMRI inputs:\n{}'.format(m)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
failed = True
break
functionals.extend(candidates)
# locate anatomical data
anat_path = os.path.join(t1_dir, os.path.basename(path), 'T1w')
files = glob.glob(anat_path + '/*')
anatomicals = []
for anat in ['aparc+aseg.nii.gz', 'aparc.a2009s+aseg.nii.gz', 'T1w_brain.nii.gz']:
if not filter(lambda x: anat in x, files):
error_message = 'expected anatomical {} not found in {}'.format(anat, anat_path)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
failed = True
break
anatomicals.append(os.path.join(anat_path, anat))
# don't run epitome if all of the inputs do not exist
if failed:
continue
# create and populate epitome directory
epi_dir = tempfile.mkdtemp()
utils.make_epitome_folders(epi_dir, len(functionals))
epi_t1_dir = '{}/TEMP/SUBJ/T1/SESS01'.format(epi_dir)
epi_func_dir = '{}/TEMP/SUBJ/FUNC/SESS01'.format(epi_dir)
try:
shutil.copyfile(anatomicals[0], '{}/anat_aparc_brain.nii.gz'.format(epi_t1_dir))
shutil.copyfile(anatomicals[1], '{}/anat_aparc2009_brain.nii.gz'.format(epi_t1_dir))
shutil.copyfile(anatomicals[2], '{}/anat_T1_brain.nii.gz'.format(epi_t1_dir))
for i, d in enumerate(functionals):
shutil.copyfile(d, '{}/RUN{}/FUNC.nii.gz'.format(epi_func_dir, '%02d' % (i + 1)))
except IOError as e:
error_message = 'unable to copy files to {}\n{}'.format(epi_dir, e)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
# collect command line options
dims = config.study_config['fmri'][exp]['dims']
tr = config.study_config['fmri'][exp]['tr']
delete = config.study_config['fmri'][exp]['del']
pipeline = config.study_config['fmri'][exp]['pipeline']
pipeline = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, 'assets/{}'.format(pipeline))
if not os.path.isfile(pipeline):
raise Exception('invalid pipeline {} defined!'.format(pipeline))
# run epitome
command = '{} {} {} {} {}'.format(pipeline, epi_dir, delete, tr, dims)
rtn, out = utils.run(command)
if rtn:
error_message = "epitome script failed: {}\n{}".format(command, out)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
else:
pass
# export fmri data
epitome_outputs = glob.glob(epi_func_dir + '/*')
for name in expected_names:
try:
matches = filter(lambda x: 'func_' + name in x, epitome_outputs)
matches.sort()
# attempt to export the defined epitome stages for all runs
if len(matches) != len(functionals):
error_message = 'epitome output {} not created for all inputs'.format(name)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
for i, match in enumerate(matches):
func_basename = utils.splitext(os.path.basename(functionals[i]))[0]
func_output = os.path.join(output_dir, func_basename + '_{}.nii.gz'.format(name))
export_file(match, func_output)
# export all anatomical / registration information
export_file_list('anat_', epitome_outputs, output_dir)
export_file_list('reg_', epitome_outputs, output_dir)
export_file_list('mat_', epitome_outputs, output_dir)
# export PARAMS folder
export_directory(os.path.join(epi_func_dir, 'PARAMS'), os.path.join(output_dir, 'PARAMS'))
except ProcessingError as p:
error_message = 'error exporting: {}'.format(p)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
# remove temporary directory
shutil.rmtree(epi_dir)
def generate_analysis_script(subject, inputs, input_type, config, study):
"""
This writes the analysis script to replicate the methods in Harvey et al
2013 Schizophrenia Bulletin. It expects timing files to exist.
Briefly, this method uses the correlation between the empathic ratings of
the participant and the actor from each video to generate an amplitude-
modulated box-car model to be fit to each time-series. This model is
convolved with an HRF, and is run alongside a standard boxcar. This allows
us to detect regions that modulate their 'activation strength' with
empathic accruacy, and those that generally track the watching of
emotionally-valenced videos (but do not parametrically modulate).
Since each video is of a different length, each block is encoded as such
in the stimulus-timing file (all times in seconds):
[start_time]*[amplitude]:[block_length]
30*5:12
See '-stim_times_AM2' in AFNI's 3dDeconvolve 'help' for more.
"""
study_base = config.get_study_base(study)
subject_dir = os.path.join(study_base, config.get_path('fmri'), 'ea', subject)
script = '{subject_dir}/{subject}_glm_1stlevel_{input_type}.sh'.format(
subject_dir=subject_dir, subject=subject, input_type=input_type)
# combine motion paramaters (glob because run does not expand * any longer)
f1 = glob.glob('{}/PARAMS/motion.*.01.1D'.format(subject_dir))[0]
f2 = glob.glob('{}/PARAMS/motion.*.02.1D'.format(subject_dir))[0]
f3 = glob.glob('{}/PARAMS/motion.*.03.1D'.format(subject_dir))[0]
rtn, out = utils.run('cat {} {} {} > {}/{}_motion.1D'.format(
f1, f2, f3, subject_dir, subject), specialquote=False)
# get input data, turn into a single string
input_list = inputs[input_type]
input_list.sort()
input_data = ''
for i in input_list:
input_data += '{} '.format(i)
# open up the master script, write common variables
f = open(script, 'wb')
f.write("""#!/bin/bash
# clean up
rm {subject_dir}/*_glm_*
# Empathic accuracy (with amplitude modulation) GLM for {sub}.
3dDeconvolve \\
-input {input_data} \\
-mask {subject_dir}/anat_EPI_mask_MNI-nonlin.nii.gz \\
-ortvec {subject_dir}/{sub}_motion.1D motion_paramaters \\
-polort 4 \\
-num_stimts 1 \\
-local_times \\
-jobs 4 \\
-x1D {subject_dir}/{sub}_glm_vid_1stlevel_design.mat \\
-stim_times_AM2 1 {subject_dir}/{sub}_vid_block-times_ea.1D \'dmBLOCK(1)\' \\
-stim_label 1 empathic_accuracy \\
-fitts {subject_dir}/{sub}_glm_vid_1stlevel_explained.nii.gz \\
-errts {subject_dir}/{sub}_glm_vid_1stlevel_residuals.nii.gz \\
-bucket {subject_dir}/{sub}_glm_vid_1stlevel.nii.gz \\
-cbucket {subject_dir}/{sub}_glm_vid_1stlevel_coeffs.nii.gz \\
-fout \\
-tout \\
-xjpeg {subject_dir}/{sub}_glm_vid_1stlevel_matrix.jpg
# Colour disciminiation (with amplitude modulation) GLM for {sub}.
3dDeconvolve \\
-input {input_data} \\
-mask {subject_dir}/anat_EPI_mask_MNI-nonlin.nii.gz \\
-ortvec {subject_dir}/{sub}_motion.1D motion_paramaters \\
-polort 4 \\
-num_stimts 1 \\
-local_times \\
-jobs 4 \\
-x1D {subject_dir}/{sub}_glm_cvid_1stlevel_design.mat \\
-stim_times_AM2 1 {subject_dir}/{sub}_cvid_block-times_ea.1D \'dmBLOCK(1)\' \\
-stim_label 1 color_videos \\
-fitts {subject_dir}/{sub}_glm_cvid_1stlevel_explained.nii.gz \\
-errts {subject_dir}/{sub}_glm_cvid_1stlevel_residuals.nii.gz \\
-bucket {subject_dir}/{sub}_glm_cvid_1stlevel.nii.gz \\
-cbucket {subject_dir}/{sub}_glm_cvid_1stlevel_coeffs.nii.gz \\
-fout \\
-tout \\
-xjpeg {subject_dir}/{sub}_glm_cvid_1stlevel_matrix.jpg
""".format(input_data=input_data, subject_dir=subject_dir, sub=subject))
f.close()
return script
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['freesurfer', 'hcp']:
if k not in config.site_config['paths']:
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
freesurfer_dir = os.path.join(study_base, config.site_config['paths']['freesurfer'])
hcp_dir = os.path.join(study_base, config.site_config['paths']['hcp'])
if scanid:
path = os.path.join(freesurfer_dir, scanid)
try:
run_hcp_convert(path, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
# run in batch mode
else:
subjects = []
freesurfer_dirs = glob.glob('{}/*'.format(freesurfer_dir))
# find subjects where at least one expected output does not exist
for path in nii_dirs:
subject = os.path.basename(path)
hcp_dir = utils.define_folder(os.path.join(study_base, config.site_config['paths']['hcp']))
if not outputs_exist(subj_dir):
subjects.append(subject)
subjects = list(set(subjects))
# submit a list of calls to ourself, one per subject
commands = []
if debug:
debugopt = '--debug'
else:
debugopt = ''
for subject in subjects:
commands.append(" ".join([__file__, study, '--subject {} '.format(subject), debugopt]))
if commands:
logger.debug('queueing up the following commands:\n'+'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_fs2hcp_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
#rtn, out = utils.run('echo bash -l {}/{} {} | qbatch -N {} --logdir {} --walltime {} -'.format(bin_dir, script, subid, jobname, logs_dir, walltime))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def main():
"""
Runs .nrrd data through unring.py.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
batch = arguments['--batch']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['nrrd']:
if k not in config.get_path('Paths'):
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
nrrd_dir = os.path.join(study_base, config.get_path('nrrd'))
# runs in serial (due to MATLAB dependencies)
if batch:
try:
run_all(nrrd_dir, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
# default behaviour: submit self to queue in batch mode
else:
if debug:
debugopt = '--debug'
else:
debugopt = ''
cmd = 'python {} {} --batch {}'.format(__file__, study, debugopt)
jobname = 'dm_unring_{}'.format(time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error(
'study {} not defined in master configuration file\n{}'.format(
study, NODE))
sys.exit(1)
study_base = config.get_study_base(study)
if 'fmri' not in config.site_config['paths']:
logger.error(
"paths:fmri not defined in site configuration file\n{}".format(
NODE))
sys.exit(1)
fmri_dir = os.path.join(study_base, config.site_config['paths']['fmri'])
if scanid:
path = os.path.join(fmri_dir, scanid)
try:
run_analysis(scanid, config, study)
except Exception as e:
logger.error(e)
sys.exit(1)
# run in batch mode
else:
# look for subjects with at least one fmri type missing outputs
subjects = []
# loop through fmri experiments defined
for exp in config.study_config['fmri'].keys():
expected_files = config.study_config['fmri'][exp]['conn']
fmri_dirs = glob.glob('{}/*'.format(os.path.join(fmri_dir, exp)))
for subj_dir in fmri_dirs:
candidates = glob.glob('{}/*'.format(subj_dir))
for filetype in expected_files:
# add subject if outputs don't already exist
if not filter(
lambda x: '{}_roi-corrs.csv'.format(filetype) in x,
candidates):
subjects.append(os.path.basename(subj_dir))
break
# collapse found subjects (do not double-count) and create a list of commands
commands = []
subjects = list(set(subjects))
for subject in subjects:
commands.append(" ".join(
[__file__, study, '--subject {}'.format(subject)]))
if commands:
logger.debug('queueing up the following commands:\n' +
'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_rest_{}_{}'.format(
i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run(
'qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error(
"Job submission failed. Output follows. {}".format(
NODE))
logger.error("stdout: {}".format(out))
sys.exit(1)
def generate_analysis_script(subject, inputs, input_type, config, study):
"""
This writes the analysis script to replicate the methods in Harvey et al
2013 Schizophrenia Bulletin. It expects timing files to exist.
Briefly, this method uses the correlation between the empathic ratings of
the participant and the actor from each video to generate an amplitude-
modulated box-car model to be fit to each time-series. This model is
convolved with an HRF, and is run alongside a standard boxcar. This allows
us to detect regions that modulate their 'activation strength' with
empathic accruacy, and those that generally track the watching of
emotionally-valenced videos (but do not parametrically modulate).
Since each video is of a different length, each block is encoded as such
in the stimulus-timing file (all times in seconds):
[start_time]*[amplitude]:[block_length]
30*5:12
See '-stim_times_AM2' in AFNI's 3dDeconvolve 'help' for more.
"""
study_base = config.get_study_base(study)
subject_dir = os.path.join(study_base, config.site_config['paths']['fmri'],
'ea', subject)
script = '{subject_dir}/{subject}_glm_1stlevel_{input_type}.sh'.format(
subject_dir=subject_dir, subject=subject, input_type=input_type)
# combine motion paramaters (glob because run does not expand * any longer)
f1 = glob.glob('{}/PARAMS/motion.*.01.1D'.format(subject_dir))[0]
f2 = glob.glob('{}/PARAMS/motion.*.02.1D'.format(subject_dir))[0]
f3 = glob.glob('{}/PARAMS/motion.*.03.1D'.format(subject_dir))[0]
rtn, out = utils.run('cat {} {} {} > {}/{}_motion.1D'.format(
f1, f2, f3, subject_dir, subject),
specialquote=False)
# get input data, turn into a single string
input_list = inputs[input_type]
input_list.sort()
input_data = ''
for i in input_list:
input_data += '{} '.format(i)
# open up the master script, write common variables
f = open(script, 'wb')
f.write("""#!/bin/bash
# clean up
rm {subject_dir}/*_glm_*
# Empathic accuracy (with amplitude modulation) GLM for {sub}.
3dDeconvolve \\
-input {input_data} \\
-mask {subject_dir}/anat_EPI_mask_MNI-nonlin.nii.gz \\
-ortvec {subject_dir}/{sub}_motion.1D motion_paramaters \\
-polort 4 \\
-num_stimts 1 \\
-local_times \\
-jobs 4 \\
-x1D {subject_dir}/{sub}_glm_vid_1stlevel_design.mat \\
-stim_times_AM2 1 {subject_dir}/{sub}_vid_block-times_ea.1D \'dmBLOCK(1)\' \\
-stim_label 1 empathic_accuracy \\
-fitts {subject_dir}/{sub}_glm_vid_1stlevel_explained.nii.gz \\
-errts {subject_dir}/{sub}_glm_vid_1stlevel_residuals.nii.gz \\
-bucket {subject_dir}/{sub}_glm_vid_1stlevel.nii.gz \\
-cbucket {subject_dir}/{sub}_glm_vid_1stlevel_coeffs.nii.gz \\
-fout \\
-tout \\
-xjpeg {subject_dir}/{sub}_glm_vid_1stlevel_matrix.jpg
# Colour disciminiation (with amplitude modulation) GLM for {sub}.
3dDeconvolve \\
-input {input_data} \\
-mask {subject_dir}/anat_EPI_mask_MNI-nonlin.nii.gz \\
-ortvec {subject_dir}/{sub}_motion.1D motion_paramaters \\
-polort 4 \\
-num_stimts 1 \\
-local_times \\
-jobs 4 \\
-x1D {subject_dir}/{sub}_glm_cvid_1stlevel_design.mat \\
-stim_times_AM2 1 {subject_dir}/{sub}_cvid_block-times_ea.1D \'dmBLOCK(1)\' \\
-stim_label 1 color_videos \\
-fitts {subject_dir}/{sub}_glm_cvid_1stlevel_explained.nii.gz \\
-errts {subject_dir}/{sub}_glm_cvid_1stlevel_residuals.nii.gz \\
-bucket {subject_dir}/{sub}_glm_cvid_1stlevel.nii.gz \\
-cbucket {subject_dir}/{sub}_glm_cvid_1stlevel_coeffs.nii.gz \\
-fout \\
-tout \\
-xjpeg {subject_dir}/{sub}_glm_cvid_1stlevel_matrix.jpg
""".format(input_data=input_data, subject_dir=subject_dir, sub=subject))
f.close()
return script
def main():
"""
Runs fmri data through the specified epitome script.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
output = arguments['--output']
exports = arguments['--exports']
task = arguments['--task']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
#Parse optional arguments
output_dir = output if output else os.path.join(study_base,config.get_path('fmri'))
opt_exports = [e for e in exports.split(',')] if exports else []
#Check if task is available
if task:
try:
config.study_config['fmri'][task]
except KeyError:
logger.error('Task {} not found in study config!'.format(task))
sys.exit(1)
tasks = {k:v for k,v in config.study_config['fmri'].iteritems() if k == task}
else:
tasks = config.study_config['fmri']
for k in ['nii', 'fmri', 'hcp']:
if k not in config.get_key('Paths'):
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
for x in tasks.iteritems():
for k in ['dims', 'del', 'pipeline', 'tags', 'export', 'tr']:
if k not in x[1].keys():
logger.error("fmri:{}:{} not defined in configuration file".format(x[0], k))
sys.exit(1)
nii_dir = os.path.join(study_base, config.get_path('nii'))
if scanid:
path = os.path.join(nii_dir, scanid)
if '_PHA_' in scanid:
sys.exit('Subject {} if a phantom, cannot be analyzed'.format(scanid))
try:
run_epitome(path, config, study, output_dir, opt_exports,tasks)
except Exception as e:
logging.error(e)
sys.exit(1)
# run in batch mode
else:
subjects = []
nii_dirs = glob.glob('{}/*'.format(nii_dir))
# find subjects where at least one expected output does not exist
for path in nii_dirs:
subject = os.path.basename(path)
if sid.is_phantom(subject):
logger.debug("Subject {} is a phantom. Skipping.".format(subject))
continue
fmri_dir = utils.define_folder(output_dir)
for exp in config.study_config['fmri'].keys():
expected_names = set(config.study_config['fmri'][exp]['export'] + opt_exports)
subj_dir = os.path.join(fmri_dir, exp, subject)
if not outputs_exist(subj_dir, expected_names):
subjects.append(subject)
break
subjects = list(set(subjects))
# submit a list of calls to ourself, one per subject
commands = []
g_opts = ' --output {} --exports {}'.format(output_dir,exports)
if task:
g_opts += ' --task {}'.format(task)
if debug:
g_opts += ' --debug'
for subject in subjects:
sub_tag = ' --subject {}'.format(subject)
commands.append(" ".join(['python ', __file__, study,g_opts,sub_tag]))
if commands:
logger.debug('queueing up the following commands:\n'+'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_fmri_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def run_epitome(path, config, study, output, exports, tasks):
"""
Finds the appropriate inputs for input subject, builds a temporary epitome
folder, runs epitome, and finally copies the outputs to the fmri_dir.
"""
study_base = config.get_study_base(study)
subject = os.path.basename(path)
nii_dir = os.path.join(study_base, config.get_path('nii'))
t1_dir = os.path.join(study_base, config.get_path('hcp'))
fmri_dir = utils.define_folder(output)
experiments = tasks.keys()
# run file collection --> epitome --> export for each study
logger.debug('experiments found {}'.format(experiments))
for exp in experiments:
logger.debug('running experiment {}'.format(exp))
# collect the files needed for each experiment
expected_names = set(config.study_config['fmri'][exp]['export'] + exports)
expected_tags = config.study_config['fmri'][exp]['tags']
output_dir = utils.define_folder(os.path.join(fmri_dir, exp, subject))
# don't run if the outputs of epitome already exist
if outputs_exist(output_dir, expected_names):
continue
# reset / remove error.log
error_log = os.path.join(output_dir, 'error.log')
if os.path.isfile(error_log):
os.remove(error_log)
failed = False
if type(expected_tags) == str:
expected_tags = [expected_tags]
# locate functional data
files = glob.glob(path + '/*')
functionals = []
for tag in expected_tags:
candidates = filter(lambda x: tag in x, files)
candidates = utils.filter_niftis(candidates)
candidates.sort()
logger.debug('checking functional inputs {}'.format(candidates))
try:
check_inputs(config, tag, path, candidates)
except Exception as m:
error_message = 'Did not find the correct number of fMRI inputs:\n{}'.format(m)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
failed = True
break
functionals.extend(candidates)
# locate anatomical data
anat_path = os.path.join(t1_dir, os.path.basename(path), 'T1w')
files = glob.glob(anat_path + '/*')
anatomicals = []
for anat in ['aparc+aseg.nii.gz', 'aparc.a2009s+aseg.nii.gz', 'T1w_brain.nii.gz']:
if not filter(lambda x: anat in x, files):
error_message = 'expected anatomical {} not found in {}'.format(anat, anat_path)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
failed = True
break
anatomicals.append(os.path.join(anat_path, anat))
# don't run epitome if all of the inputs do not exist
if failed:
continue
# create and populate epitome directory
epi_dir = tempfile.mkdtemp()
make_epitome_folders(epi_dir, len(functionals))
epi_t1_dir = '{}/TEMP/SUBJ/T1/SESS01'.format(epi_dir)
epi_func_dir = '{}/TEMP/SUBJ/FUNC/SESS01'.format(epi_dir)
try:
shutil.copyfile(anatomicals[0], '{}/anat_aparc_brain.nii.gz'.format(epi_t1_dir))
shutil.copyfile(anatomicals[1], '{}/anat_aparc2009_brain.nii.gz'.format(epi_t1_dir))
shutil.copyfile(anatomicals[2], '{}/anat_T1_brain.nii.gz'.format(epi_t1_dir))
for i, d in enumerate(functionals):
shutil.copyfile(d, '{}/RUN{}/FUNC.nii.gz'.format(epi_func_dir, '%02d' % (i + 1)))
except IOError as e:
error_message = 'unable to copy files to {}\n{}'.format(epi_dir, e)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
# collect command line options
dims = config.study_config['fmri'][exp]['dims']
tr = config.study_config['fmri'][exp]['tr']
delete = config.study_config['fmri'][exp]['del']
pipeline = config.study_config['fmri'][exp]['pipeline']
pipeline = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, 'assets/{}'.format(pipeline))
if not os.path.isfile(pipeline):
raise Exception('invalid pipeline {} defined!'.format(pipeline))
# run epitome
command = '{} {} {} {} {}'.format(pipeline, epi_dir, delete, tr, dims)
rtn, out = utils.run(command)
if rtn:
error_message = "epitome script failed: {}\n{}".format(command, out)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
else:
pass
# export fmri data
epitome_outputs = glob.glob(epi_func_dir + '/*')
for name in expected_names:
try:
matches = filter(lambda x: 'func_' + name in x, epitome_outputs)
matches.sort()
# attempt to export the defined epitome stages for all runs
if len(matches) != len(functionals):
error_message = 'epitome output {} not created for all inputs'.format(name)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
for i, match in enumerate(matches):
func_basename = utils.splitext(os.path.basename(functionals[i]))[0]
func_output = os.path.join(output_dir, func_basename + '_{}.nii.gz'.format(name))
export_file(match, func_output)
# export all anatomical / registration information
export_file_list('anat_', epitome_outputs, output_dir)
export_file_list('reg_', epitome_outputs, output_dir)
export_file_list('mat_', epitome_outputs, output_dir)
# export PARAMS folder
export_directory(os.path.join(epi_func_dir, 'PARAMS'), os.path.join(output_dir, 'PARAMS'))
except ProcessingError as p:
error_message = 'error exporting: {}'.format(p)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
# remove temporary directory
shutil.rmtree(epi_dir)
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
subject = arguments['--subject']
debug = arguments['--debug']
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
if 'ea' not in config.study_config['fmri'].keys():
logger.error('ea not defined in fmri in {}'.format(config_file))
sys.exit(1)
for k in ['nii', 'fmri', 'hcp']:
if k not in config.site_config['paths']:
logger.error("paths:{} not defined in {}".format(k, config_file))
sys.exit(1)
ea_dir = os.path.join(study_base, config.site_config['paths']['fmri'],
'ea')
nii_dir = os.path.join(study_base, config.site_config['paths']['nii'])
if subject:
if '_PHA_' in subject:
logger.error("{} is a phantom, cannot analyze".format(subject))
sys.exit(1)
analyze_subject(subject, config, study)
else:
# batch mode
subjects = glob.glob('{}/*'.format(nii_dir))
commands = []
if debug:
opts = '--debug'
else:
opts = ''
for path in subjects:
subject = os.path.basename(path)
if check_complete(ea_dir, subject):
logger.debug('{} already analysed'.format(subject))
else:
commands.append(" ".join(
[__file__, study, '--subject {}'.format(subject), opts]))
if commands:
logger.debug("queueing up the following commands:\n" +
'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = "dm_ea_{}_{}".format(i,
time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run(
'qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
# qbacth method -- might bring it back, but not needed
#fd, path = tempfile.mkstemp()
#os.write(fd, '\n'.join(commands))
#os.close(fd)
#rtn, out, err = utils.run('qbatch -i --logdir {ld} -N {name} --walltime {wt} {cmds}'.format(ld=logdir, name=jobname, wt=walltime, cmds=path))
if rtn:
logger.error(
"Job submission failed\nstdout: {}".format(out))
sys.exit(1)
msg = 'Failed to open block_times & corr_push for {} with excuse {}'.format(subject, e.strerror)
logger.error(msg)
finally:
f1.close()
f2.close()
f3.close()
f4.close()
# run the GLM
files = glob.glob(os.path.join(ea_dir, subject + '/*.nii.gz'))
inputs = get_inputs(files, config)
for input_type in inputs.keys():
script = generate_analysis_script(subject, inputs, input_type, config, study)
rtn, out = utils.run('chmod 754 {}'.format(script))
rtn, out = utils.run(script)
if rtn:
logger.error('Script {} failed to run on subject {} with error:\n{}'.format(
script, subject, out))
sys.exit(1)
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
subject = arguments['--subject']
debug = arguments['--debug']
logging.info('Starting')
if debug:
def run_analysis(scanid, config, study):
"""
Extracts: time series, correlation matricies using defined atlas.
"""
study_base = config.get_study_base(study)
fmri_dir = os.path.join(study_base, config.site_config['paths']['fmri'])
experiments = config.study_config['fmri'].keys()
atlas = os.path.join(os.path.dirname(os.path.realpath(__file__)),
os.pardir, 'assets/shen_2mm_268_parcellation.nii.gz')
if not os.path.isfile(atlas):
print('ERROR: atlas file {} not found'.format(atlas))
sys.exit(1)
for exp in experiments:
path = os.path.join(fmri_dir, exp, scanid)
# get filetypes to analyze, ignoring ROI files
inputs = get_inputs(config, path, exp, scanid)
for filename in inputs:
basename = os.path.basename(utils.splitext(filename)[0])
# if the final correlation matrix exists, skip processing
if os.path.isfile(os.path.join(path, basename + '_roi-corrs.csv')):
continue
# generate ROI file in register with subject's data
roi_file = os.path.join(path, basename + '_rois.nii.gz')
if not os.path.isfile(roi_file):
rtn, out = utils.run(
'3dresample -master {} -prefix {} -inset {}'.format(
filename, roi_file, atlas))
if rtn:
logger.error('{}\n{}'.format(out, NODE))
raise Exception(
'Error resampling atlas {} to match {}.'.format(
atlas, filename))
else:
pass
rois, _, _, _ = utils.loadnii(roi_file)
data, _, _, _ = utils.loadnii(filename)
n_rois = len(np.unique(rois[rois > 0]))
dims = np.shape(data)
# loop through all ROIs, extracting mean timeseries.
output = np.zeros((n_rois, dims[1]))
for i, roi in enumerate(np.unique(rois[rois > 0])):
idx = np.where(rois == roi)[0]
if len(idx) > 0:
output[i, :] = np.mean(data[idx, :], axis=0)
# save the raw time series
np.savetxt(os.path.join(path, basename + '_roi-timeseries.csv'),
output,
delimiter=',')
# save the full correlation matrix
corrs = np.corrcoef(output)
np.savetxt(os.path.join(path, basename + '_roi-corrs.csv'),
corrs,
delimiter=',')
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
subject = arguments['--subject']
debug = arguments['--debug']
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
if 'ea' not in config.study_config['fmri'].keys():
logger.error('ea not defined in fmri in {}'.format(config_file))
sys.exit(1)
for k in ['nii', 'fmri', 'hcp']:
if k not in config.get_key('Paths'):
logger.error("paths:{} not defined in {}".format(k, config_file))
sys.exit(1)
ea_dir = os.path.join(study_base, config.get_path('fmri'), 'ea')
nii_dir = os.path.join(study_base, config.get_path('nii'))
if subject:
if '_PHA_' in subject:
logger.error("{} is a phantom, cannot analyze".format(subject))
sys.exit(1)
analyze_subject(subject, config, study)
else:
# batch mode
subjects = glob.glob('{}/*'.format(nii_dir))
commands = []
if debug:
opts = '--debug'
else:
opts = ''
for path in subjects:
subject = os.path.basename(path)
if check_complete(ea_dir, subject):
logger.debug('{} already analysed'.format(subject))
else:
commands.append(" ".join([__file__, study, '--subject {}'.format(subject), opts]))
if commands:
logger.debug("queueing up the following commands:\n"+'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = "dm_ea_{}_{}".format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
# qbacth method -- might bring it back, but not needed
#fd, path = tempfile.mkstemp()
#os.write(fd, '\n'.join(commands))
#os.close(fd)
#rtn, out, err = utils.run('qbatch -i --logdir {ld} -N {name} --walltime {wt} {cmds}'.format(ld=logdir, name=jobname, wt=walltime, cmds=path))
if rtn:
logger.error("Job submission failed\nstdout: {}".format(out))
sys.exit(1)
def main():
"""
Runs .nrrd data through unring.py.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
batch = arguments['--batch']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['nrrd']:
if k not in config.get_path('Paths'):
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
nrrd_dir = os.path.join(study_base, config.get_path('nrrd'))
# runs in serial (due to MATLAB dependencies)
if batch:
try:
run_all(nrrd_dir, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
# default behaviour: submit self to queue in batch mode
else:
if debug:
debugopt = '--debug'
else:
debugopt = ''
cmd = 'python {} {} --batch {}'.format(__file__, study, debugopt)
jobname = 'dm_unring_{}'.format(time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run(
'qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def main():
"""
Loops through subjects, preprocessing using supplied script, and runs a
first-level GLM using AFNI (tent functions, 15 s window) on all subjects.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
subject = arguments['--subject']
debug = arguments['--debug']
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
imob_dir = os.path.join(study_base, config.site_config['paths']['fmri'],
'imob')
# process a single subject
if subject:
# get required inputs from each
files = glob.glob(os.path.join(imob_dir, subject) + '/*.nii.gz')
inputs = get_inputs(files, config)
# check if subject has already been processed
if check_complete(imob_dir, subject):
logger.info('{} already analysed'.format(subject))
sys.exit(0)
# first level GLM for inputs
for input_type in inputs.keys():
script = generate_analysis_script(subject, inputs, input_type,
config, study)
rtn, out = utils.run('chmod 754 {}'.format(script))
rtn, out = utils.run(script)
if rtn:
logger.error(
'Script {} failed to run on subject {} with error:\n{}'.
format(script, subject, out))
sys.exit(1)
# process all subjects
else:
commands = []
for path in glob.glob('{}/*'.format(imob_dir)):
subject = os.path.basename(path)
# add subject if any of the expected outputs do not exist
files = glob.glob(os.path.join(imob_dir, subject) + '/*.nii.gz')
try:
inputs = get_inputs(files, config)
except:
logger.debug('Invalid inputs for {}'.format(subject))
continue
expected = inputs.keys()
for exp in expected:
if not filter(
lambda x: '{}_glm_IM_1stlvl_{}'.format(subject, exp) in
x, files):
commands.append(" ".join(
[__file__, study, '--subject {}'.format(subject)]))
break
if commands:
logger.debug("queueing up the following commands:\n" +
'\n'.join(commands))
#fd, path = tempfile.mkstemp()
#os.write(fd, '\n'.join(commands))
#os.close(fd)
for i, cmd in enumerate(commands):
jobname = "dm_imob_{}_{}".format(
i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run(
'qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
#rtn, out, err = utils.run('qbatch -i --logdir {logdir} -N {name} --walltime {wt} {cmds}'.format(logdir = log_path, name = jobname, wt = walltime, cmds = path))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
