def run_pipeline(config, subject, t1, t2):
if not input_exists(t1) or not input_exists(t2):
sys.exit(1)
base_dir = utils.define_folder(config.get_path('hcp_fs'))
dest_dir = utils.define_folder(os.path.join(base_dir, subject))
with utils.cd(dest_dir):
hcp_pipeline = "hcp-freesurfer.sh {} {} {} {}".format(
base_dir, subject, t1, t2)
rtn, out = utils.run(hcp_pipeline, dryrun=DRYRUN)
if rtn:
logger.error("hcp-freesurfer.sh exited with non-zero status code. "
"Output: {}".format(out))
def run_pipeline(config, subject, t1, t2):
if not input_exists(t1) or not input_exists(t2):
sys.exit(1)
base_dir = utils.define_folder(config.get_path('hcp_fs'))
dest_dir = utils.define_folder(os.path.join(base_dir, subject))
with utils.cd(dest_dir):
hcp_pipeline = "hcp-freesurfer.sh {} {} {} {}".format(base_dir, subject,
t1, t2)
rtn, out = utils.run(hcp_pipeline, dryrun=DRYRUN)
if rtn:
logger.error("hcp-freesurfer.sh exited with non-zero status code. "
"Output: {}".format(out))
def run_hcp_convert(path, config, study):
"""Runs fs2hcp on the input subject"""
study_base = config.get_study_base(study)
subject = os.path.basename(path)
freesurfer_dir = os.path.join(study_base, config.site_config['paths']['freesurfer'])
hcp_dir = os.path.join(study_base, config.site_config['paths']['hcp'])
output_dir = utils.define_folder(os.path.join(hcp_dir, subject))
if outputs_exist(output_dir):
logger.debug('outputs found in {}'.format(path))
sys.exit()
# reset / remove error.log
error_log = os.path.join(output_dir, 'error.log')
if os.path.isfile(error_log):
os.remove(error_log)
# run fs2hcp
command = 'fs2hcp --FSpath={} --HCPpath={} --subject={}'.format(freesurfer_dir, hcp_dir, subject)
rtn, out = utils.run(command)
if rtn:
error_message = "fs2hcp failed: {}\n{}".format(command, out)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
def get_new_subjects(config, qc_subjects):
fs_subjects = []
for subject in qc_subjects:
if sid.is_phantom(subject):
logger.debug("Subject {} is a phantom. Skipping.".format(subject))
continue
freesurfer_dir = utils.define_folder(config.get_path('freesurfer'))
fs_subject_dir = os.path.join(freesurfer_dir, subject)
if not outputs_exist(fs_subject_dir):
fs_subjects.append(subject)
return fs_subjects
def get_new_subjects(config, qc_subjects):
fs_subjects = []
for subject in qc_subjects:
if sid.is_phantom(subject):
logger.debug("Subject {} is a phantom. Skipping.".format(subject))
continue
freesurfer_dir = utils.define_folder(config.get_path('freesurfer'))
fs_subject_dir = os.path.join(freesurfer_dir, subject)
if not outputs_exist(fs_subject_dir):
fs_subjects.append(subject)
return fs_subjects
def run_all(nrrd_dir, config, study):
"""Finds all non-phantom input nrrds and run unring.py in serial."""
study_base = config.get_study_base(study)
subjects = os.listdir(nrrd_dir)
subjects = filter(lambda x: '_PHA_' not in x, subjects)
unring_dir = utils.define_folder(
os.path.join(study_base, config.get_path('unring')))
tags = config.study_config['unring']['tags']
for subject in subjects:
output_dir = utils.define_folder(os.path.join(unring_dir, subject))
inputs = os.listdir(os.path.join(nrrd_dir, subject))
inputs = select_inputs(inputs,
tags) # selects inputs with matching tag
for input_file in inputs:
# don't run if the outputs of unring already exist
if outputs_exist(output_dir, input_file):
continue
# reset / remove error.log
error_log = os.path.join(output_dir, 'error.log')
if os.path.isfile(error_log):
os.remove(error_log)
output_fname = os.path.join(output_dir, input_file)
input_fname = os.path.join(nrrd_dir, subject, input_file)
cmd = 'unring.py {} {} -v'.format(input_fname, output_fname)
logger.debug('running {}'.format(cmd))
rtn, out = utils.run(cmd)
if rtn:
error_message = "unring.py failed: {}\n{}".format(cmd, out)
logger.info(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
else:
pass
def run_all(nrrd_dir, config, study):
"""Finds all non-phantom input nrrds and run unring.py in serial."""
study_base = config.get_study_base(study)
subjects = os.listdir(nrrd_dir)
subjects = filter(lambda x: '_PHA_' not in x, subjects)
unring_dir = utils.define_folder(os.path.join(study_base, config.get_path('unring')))
tags = config.study_config['unring']['tags']
for subject in subjects:
output_dir = utils.define_folder(os.path.join(unring_dir, subject))
inputs = os.listdir(os.path.join(nrrd_dir, subject))
inputs = select_inputs(inputs, tags) # selects inputs with matching tag
for input_file in inputs:
# don't run if the outputs of unring already exist
if outputs_exist(output_dir, input_file):
continue
# reset / remove error.log
error_log = os.path.join(output_dir, 'error.log')
if os.path.isfile(error_log):
os.remove(error_log)
output_fname = os.path.join(output_dir, input_file)
input_fname = os.path.join(nrrd_dir, subject, input_file)
cmd = 'unring.py {} {} -v'.format(input_fname, output_fname)
logger.debug('running {}'.format(cmd))
rtn, out = utils.run(cmd)
if rtn:
error_message = "unring.py failed: {}\n{}".format(cmd, out)
logger.info(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
else:
pass
def main():
"""
Runs fmri data through the specified epitome script.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['nii', 'fmri', 'hcp']:
if k not in config.site_config['paths']:
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
for x in config.study_config['fmri'].iteritems():
for k in ['dims', 'del', 'pipeline', 'tags', 'export', 'tr']:
if k not in x[1].keys():
logger.error("fmri:{}:{} not defined in configuration file".format(x[0], k))
sys.exit(1)
nii_dir = os.path.join(study_base, config.site_config['paths']['nii'])
if scanid:
path = os.path.join(nii_dir, scanid)
if '_PHA_' in scanid:
sys.exit('Subject {} if a phantom, cannot be analyzed'.format(scanid))
try:
run_epitome(path, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
# run in batch mode
else:
subjects = []
nii_dirs = glob.glob('{}/*'.format(nii_dir))
# find subjects where at least one expected output does not exist
for path in nii_dirs:
subject = os.path.basename(path)
if sid.is_phantom(subject):
logger.debug("Subject {} is a phantom. Skipping.".format(subject))
continue
fmri_dir = utils.define_folder(os.path.join(study_base, config.site_config['paths']['fmri']))
for exp in config.study_config['fmri'].keys():
expected_names = config.study_config['fmri'][exp]['export']
subj_dir = os.path.join(fmri_dir, exp, subject)
if not outputs_exist(subj_dir, expected_names):
subjects.append(subject)
break
subjects = list(set(subjects))
# submit a list of calls to ourself, one per subject
commands = []
if debug:
debugopt = '--debug'
else:
debugopt = ''
for subject in subjects:
commands.append(" ".join(['python ', __file__, study, '--subject {} '.format(subject), debugopt]))
if commands:
logger.debug('queueing up the following commands:\n'+'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_fmri_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def run_epitome(path, config, study):
"""
Finds the appropriate inputs for input subject, builds a temporary epitome
folder, runs epitome, and finally copies the outputs to the fmri_dir.
"""
study_base = config.get_study_base(study)
subject = os.path.basename(path)
nii_dir = os.path.join(study_base, config.site_config['paths']['nii'])
t1_dir = os.path.join(study_base, config.site_config['paths']['hcp'])
fmri_dir = utils.define_folder(os.path.join(study_base, config.site_config['paths']['fmri']))
experiments = config.study_config['fmri'].keys()
# run file collection --> epitome --> export for each study
logger.debug('experiments found {}'.format(experiments))
for exp in experiments:
logger.debug('running experiment {}'.format(exp))
# collect the files needed for each experiment
expected_names = config.study_config['fmri'][exp]['export']
expected_tags = config.study_config['fmri'][exp]['tags']
output_dir = utils.define_folder(os.path.join(fmri_dir, exp, subject))
# don't run if the outputs of epitome already exist
if outputs_exist(output_dir, expected_names):
continue
# reset / remove error.log
error_log = os.path.join(output_dir, 'error.log')
if os.path.isfile(error_log):
os.remove(error_log)
failed = False
if type(expected_tags) == str:
expected_tags = [expected_tags]
# locate functional data
files = glob.glob(path + '/*')
functionals = []
for tag in expected_tags:
candidates = filter(lambda x: tag in x, files)
candidates = utils.filter_niftis(candidates)
candidates.sort()
logger.debug('checking functional inputs {}'.format(candidates))
try:
check_inputs(config, tag, path, candidates)
except Exception as m:
error_message = 'Did not find the correct number of fMRI inputs:\n{}'.format(m)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
failed = True
break
functionals.extend(candidates)
# locate anatomical data
anat_path = os.path.join(t1_dir, os.path.basename(path), 'T1w')
files = glob.glob(anat_path + '/*')
anatomicals = []
for anat in ['aparc+aseg.nii.gz', 'aparc.a2009s+aseg.nii.gz', 'T1w_brain.nii.gz']:
if not filter(lambda x: anat in x, files):
error_message = 'expected anatomical {} not found in {}'.format(anat, anat_path)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
failed = True
break
anatomicals.append(os.path.join(anat_path, anat))
# don't run epitome if all of the inputs do not exist
if failed:
continue
# create and populate epitome directory
epi_dir = tempfile.mkdtemp()
utils.make_epitome_folders(epi_dir, len(functionals))
epi_t1_dir = '{}/TEMP/SUBJ/T1/SESS01'.format(epi_dir)
epi_func_dir = '{}/TEMP/SUBJ/FUNC/SESS01'.format(epi_dir)
try:
shutil.copyfile(anatomicals[0], '{}/anat_aparc_brain.nii.gz'.format(epi_t1_dir))
shutil.copyfile(anatomicals[1], '{}/anat_aparc2009_brain.nii.gz'.format(epi_t1_dir))
shutil.copyfile(anatomicals[2], '{}/anat_T1_brain.nii.gz'.format(epi_t1_dir))
for i, d in enumerate(functionals):
shutil.copyfile(d, '{}/RUN{}/FUNC.nii.gz'.format(epi_func_dir, '%02d' % (i + 1)))
except IOError as e:
error_message = 'unable to copy files to {}\n{}'.format(epi_dir, e)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
# collect command line options
dims = config.study_config['fmri'][exp]['dims']
tr = config.study_config['fmri'][exp]['tr']
delete = config.study_config['fmri'][exp]['del']
pipeline = config.study_config['fmri'][exp]['pipeline']
pipeline = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, 'assets/{}'.format(pipeline))
if not os.path.isfile(pipeline):
raise Exception('invalid pipeline {} defined!'.format(pipeline))
# run epitome
command = '{} {} {} {} {}'.format(pipeline, epi_dir, delete, tr, dims)
rtn, out = utils.run(command)
if rtn:
error_message = "epitome script failed: {}\n{}".format(command, out)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
else:
pass
# export fmri data
epitome_outputs = glob.glob(epi_func_dir + '/*')
for name in expected_names:
try:
matches = filter(lambda x: 'func_' + name in x, epitome_outputs)
matches.sort()
# attempt to export the defined epitome stages for all runs
if len(matches) != len(functionals):
error_message = 'epitome output {} not created for all inputs'.format(name)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
for i, match in enumerate(matches):
func_basename = utils.splitext(os.path.basename(functionals[i]))[0]
func_output = os.path.join(output_dir, func_basename + '_{}.nii.gz'.format(name))
export_file(match, func_output)
# export all anatomical / registration information
export_file_list('anat_', epitome_outputs, output_dir)
export_file_list('reg_', epitome_outputs, output_dir)
export_file_list('mat_', epitome_outputs, output_dir)
# export PARAMS folder
export_directory(os.path.join(epi_func_dir, 'PARAMS'), os.path.join(output_dir, 'PARAMS'))
except ProcessingError as p:
error_message = 'error exporting: {}'.format(p)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
# remove temporary directory
shutil.rmtree(epi_dir)
def main():
"""
Runs fmri data through the specified epitome script.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
output = arguments['--output']
exports = arguments['--exports']
task = arguments['--task']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
#Parse optional arguments
output_dir = output if output else os.path.join(study_base,config.get_path('fmri'))
opt_exports = [e for e in exports.split(',')] if exports else []
#Check if task is available
if task:
try:
config.study_config['fmri'][task]
except KeyError:
logger.error('Task {} not found in study config!'.format(task))
sys.exit(1)
tasks = {k:v for k,v in config.study_config['fmri'].iteritems() if k == task}
else:
tasks = config.study_config['fmri']
for k in ['nii', 'fmri', 'hcp']:
if k not in config.get_key('Paths'):
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
for x in tasks.iteritems():
for k in ['dims', 'del', 'pipeline', 'tags', 'export', 'tr']:
if k not in x[1].keys():
logger.error("fmri:{}:{} not defined in configuration file".format(x[0], k))
sys.exit(1)
nii_dir = os.path.join(study_base, config.get_path('nii'))
if scanid:
path = os.path.join(nii_dir, scanid)
if '_PHA_' in scanid:
sys.exit('Subject {} if a phantom, cannot be analyzed'.format(scanid))
try:
run_epitome(path, config, study, output_dir, opt_exports,tasks)
except Exception as e:
logging.error(e)
sys.exit(1)
# run in batch mode
else:
subjects = []
nii_dirs = glob.glob('{}/*'.format(nii_dir))
# find subjects where at least one expected output does not exist
for path in nii_dirs:
subject = os.path.basename(path)
if sid.is_phantom(subject):
logger.debug("Subject {} is a phantom. Skipping.".format(subject))
continue
fmri_dir = utils.define_folder(output_dir)
for exp in config.study_config['fmri'].keys():
expected_names = set(config.study_config['fmri'][exp]['export'] + opt_exports)
subj_dir = os.path.join(fmri_dir, exp, subject)
if not outputs_exist(subj_dir, expected_names):
subjects.append(subject)
break
subjects = list(set(subjects))
# submit a list of calls to ourself, one per subject
commands = []
g_opts = ' --output {} --exports {}'.format(output_dir,exports)
if task:
g_opts += ' --task {}'.format(task)
if debug:
g_opts += ' --debug'
for subject in subjects:
sub_tag = ' --subject {}'.format(subject)
commands.append(" ".join(['python ', __file__, study,g_opts,sub_tag]))
if commands:
logger.debug('queueing up the following commands:\n'+'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_fmri_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def run_epitome(path, config, study, output, exports, tasks):
"""
Finds the appropriate inputs for input subject, builds a temporary epitome
folder, runs epitome, and finally copies the outputs to the fmri_dir.
"""
study_base = config.get_study_base(study)
subject = os.path.basename(path)
nii_dir = os.path.join(study_base, config.get_path('nii'))
t1_dir = os.path.join(study_base, config.get_path('hcp'))
fmri_dir = utils.define_folder(output)
experiments = tasks.keys()
# run file collection --> epitome --> export for each study
logger.debug('experiments found {}'.format(experiments))
for exp in experiments:
logger.debug('running experiment {}'.format(exp))
# collect the files needed for each experiment
expected_names = set(config.study_config['fmri'][exp]['export'] + exports)
expected_tags = config.study_config['fmri'][exp]['tags']
output_dir = utils.define_folder(os.path.join(fmri_dir, exp, subject))
# don't run if the outputs of epitome already exist
if outputs_exist(output_dir, expected_names):
continue
# reset / remove error.log
error_log = os.path.join(output_dir, 'error.log')
if os.path.isfile(error_log):
os.remove(error_log)
failed = False
if type(expected_tags) == str:
expected_tags = [expected_tags]
# locate functional data
files = glob.glob(path + '/*')
functionals = []
for tag in expected_tags:
candidates = filter(lambda x: tag in x, files)
candidates = utils.filter_niftis(candidates)
candidates.sort()
logger.debug('checking functional inputs {}'.format(candidates))
try:
check_inputs(config, tag, path, candidates)
except Exception as m:
error_message = 'Did not find the correct number of fMRI inputs:\n{}'.format(m)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
failed = True
break
functionals.extend(candidates)
# locate anatomical data
anat_path = os.path.join(t1_dir, os.path.basename(path), 'T1w')
files = glob.glob(anat_path + '/*')
anatomicals = []
for anat in ['aparc+aseg.nii.gz', 'aparc.a2009s+aseg.nii.gz', 'T1w_brain.nii.gz']:
if not filter(lambda x: anat in x, files):
error_message = 'expected anatomical {} not found in {}'.format(anat, anat_path)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
failed = True
break
anatomicals.append(os.path.join(anat_path, anat))
# don't run epitome if all of the inputs do not exist
if failed:
continue
# create and populate epitome directory
epi_dir = tempfile.mkdtemp()
make_epitome_folders(epi_dir, len(functionals))
epi_t1_dir = '{}/TEMP/SUBJ/T1/SESS01'.format(epi_dir)
epi_func_dir = '{}/TEMP/SUBJ/FUNC/SESS01'.format(epi_dir)
try:
shutil.copyfile(anatomicals[0], '{}/anat_aparc_brain.nii.gz'.format(epi_t1_dir))
shutil.copyfile(anatomicals[1], '{}/anat_aparc2009_brain.nii.gz'.format(epi_t1_dir))
shutil.copyfile(anatomicals[2], '{}/anat_T1_brain.nii.gz'.format(epi_t1_dir))
for i, d in enumerate(functionals):
shutil.copyfile(d, '{}/RUN{}/FUNC.nii.gz'.format(epi_func_dir, '%02d' % (i + 1)))
except IOError as e:
error_message = 'unable to copy files to {}\n{}'.format(epi_dir, e)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
# collect command line options
dims = config.study_config['fmri'][exp]['dims']
tr = config.study_config['fmri'][exp]['tr']
delete = config.study_config['fmri'][exp]['del']
pipeline = config.study_config['fmri'][exp]['pipeline']
pipeline = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, 'assets/{}'.format(pipeline))
if not os.path.isfile(pipeline):
raise Exception('invalid pipeline {} defined!'.format(pipeline))
# run epitome
command = '{} {} {} {} {}'.format(pipeline, epi_dir, delete, tr, dims)
rtn, out = utils.run(command)
if rtn:
error_message = "epitome script failed: {}\n{}".format(command, out)
logger.debug(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
else:
pass
# export fmri data
epitome_outputs = glob.glob(epi_func_dir + '/*')
for name in expected_names:
try:
matches = filter(lambda x: 'func_' + name in x, epitome_outputs)
matches.sort()
# attempt to export the defined epitome stages for all runs
if len(matches) != len(functionals):
error_message = 'epitome output {} not created for all inputs'.format(name)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
for i, match in enumerate(matches):
func_basename = utils.splitext(os.path.basename(functionals[i]))[0]
func_output = os.path.join(output_dir, func_basename + '_{}.nii.gz'.format(name))
export_file(match, func_output)
# export all anatomical / registration information
export_file_list('anat_', epitome_outputs, output_dir)
export_file_list('reg_', epitome_outputs, output_dir)
export_file_list('mat_', epitome_outputs, output_dir)
# export PARAMS folder
export_directory(os.path.join(epi_func_dir, 'PARAMS'), os.path.join(output_dir, 'PARAMS'))
except ProcessingError as p:
error_message = 'error exporting: {}'.format(p)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
continue
# remove temporary directory
shutil.rmtree(epi_dir)
def analyze_subject(subject, config, study):
"""
1) finds the behavioural log files
2) generates the stimulus timing files from these logs
3) finds the pre-processed fmri data
4) runs the standard GLM analysis on these data
"""
study_base = config.get_study_base(study)
resources_dir = os.path.join(study_base, config.get_path('resources'))
ea_dir = os.path.join(study_base, config.get_path('fmri'), 'ea')
output_dir = utils.define_folder(os.path.join(study_base, config.get_path('fmri'), 'ea', subject))
# check if subject has already been processed
if check_complete(ea_dir, subject):
msg = '{} already analysed'.format(subject)
logger.info(msg)
sys.exit(0)
# reset / remove error.log
error_log = os.path.join(output_dir, 'error.log')
if os.path.isfile(error_log):
os.remove(error_log)
# find the behavioural data, and exit if we fail to find it
try:
resdirs = glob.glob(os.path.join(resources_dir, subject + '_??'))
resources = []
for resdir in resdirs:
resfiles = [os.path.join(dp, f) for dp, dn, fn in os.walk(resdir) for f in fn]
resources.extend(resfiles)
logs = filter(lambda x: '.log' in x and 'UCLAEmpAcc' in x, resources)
logs.sort()
except:
logger.error('No BEHAV data for {}.'.format(subject))
sys.exit(1)
# if we have the wrong number of logs, don't guess which to use, just fail
if len(logs) != 3:
error_message = 'Did not find exactly 3 logs for {}\nfound:{}.'.format(subject, logs)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
sys.exit(1)
# parse and write the logs seperately for each experiment condition (video or shapes/colours video)
for test_type in ['vid','cvid']:
# extract all of the data from the logs
on_all, dur_all, corr_all, push_all, timings_all = [], [], [], [], []
try:
logger.info('Parsing {} logfiles for subject'.format(len(logs), subject))
for log in logs:
# extract the block id from the logfilename
block_id = os.path.splitext(os.path.basename(log))[0][-1]
on, dur, corr, push, timings = process_behav_data(log, output_dir, subject, test_type, block_id)
on_all.extend(on)
dur_all.extend(dur)
corr_all.extend(corr)
push_all.extend(push)
timings_all.extend(timings)
except Exception, e:
msg = 'Failed to parse logs for {}, with {}.'.format(subject, str(e))
logger.error(msg)
sys.exit(1)
# write data to stimulus timing file for AFNI, and a QC csv
# on_all = sorted(on_all, key=lambda x:x[1])
timings_all = sorted(timings_all, key=lambda x: (x[2], x[3], x[1])) # put the responses into order
try:
logger.info('Writing stimulus data')
# write each stimulus time:
# [start_time]*[amplitude],[buttonpushes]:[block_length]
# 30*5,0.002:12
# OFFSET 4 TRs == 8 Seconds!
# on = on - 8.0
f1 = open('{}/{}_{}_block-times_ea.1D'.format(output_dir, subject, test_type), 'wb') # stim timing file
f2 = open('{}/{}_{}_corr_push.csv'.format(output_dir, subject, test_type), 'wb') # r values and num pushes / minute
f3 = open('{}/{}_{}_button-times.csv'.format(output_dir, subject, test_type), 'wb') # button responses and timings
f4 = open('{}/{}_{}_vid-onsets.csv'.format(output_dir, subject, test_type), 'wb') # button responses and timings
f2.write('correlation,n-pushes-per-minute\n')
f3.write('Block_ID,Video,Response,Timing\n')
f4.write('Block_ID,Video, Onset\n')
for i in range(len(on_all)):
f1.write('{o:.2f}*{r:.2f},{p}:{d:.2f} '.format(o=on_all[i][1]-8.0, r=corr_all[i], p=push_all[i], d=dur_all[i]))
f2.write('{r:.2f},{p}\n'.format(r=corr_all[i], p=push_all[i]))
for timing in timings_all:
f3.write('{b},{v},{r},{t:.2f}\n'.format(b=timing[2], v=timing[3], r=timing[0], t=timing[1]))
for onset in on_all:
f4.write('{b},{r},{t:.2f}\n'.format(b=onset[2], r=onset[0], t=onset[1]))
f1.write('\n') # add newline at the end of each run (up to 3 runs.)
except IOError as e:
msg = 'Failed to open block_times & corr_push for {} with excuse {}'.format(subject, e.strerror)
logger.error(msg)
finally:
f1.close()
f2.close()
f3.close()
f4.close()
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['freesurfer', 'hcp']:
if k not in config.site_config['paths']:
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
freesurfer_dir = os.path.join(study_base, config.site_config['paths']['freesurfer'])
hcp_dir = os.path.join(study_base, config.site_config['paths']['hcp'])
if scanid:
path = os.path.join(freesurfer_dir, scanid)
try:
run_hcp_convert(path, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
# run in batch mode
else:
subjects = []
freesurfer_dirs = glob.glob('{}/*'.format(freesurfer_dir))
# find subjects where at least one expected output does not exist
for path in nii_dirs:
subject = os.path.basename(path)
hcp_dir = utils.define_folder(os.path.join(study_base, config.site_config['paths']['hcp']))
if not outputs_exist(subj_dir):
subjects.append(subject)
subjects = list(set(subjects))
# submit a list of calls to ourself, one per subject
commands = []
if debug:
debugopt = '--debug'
else:
debugopt = ''
for subject in subjects:
commands.append(" ".join([__file__, study, '--subject {} '.format(subject), debugopt]))
if commands:
logger.debug('queueing up the following commands:\n'+'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_fs2hcp_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
#rtn, out = utils.run('echo bash -l {}/{} {} | qbatch -N {} --logdir {} --walltime {} -'.format(bin_dir, script, subid, jobname, logs_dir, walltime))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def analyze_subject(subject, config, study):
"""
1) finds the behavioural log files
2) generates the stimulus timing files from these logs
3) finds the pre-processed fmri data
4) runs the standard GLM analysis on these data
"""
study_base = config.get_study_base(study)
resources_dir = os.path.join(study_base,
config.site_config['paths']['resources'])
ea_dir = os.path.join(study_base, config.site_config['paths']['fmri'],
'ea')
output_dir = utils.define_folder(
os.path.join(study_base, config.site_config['paths']['fmri'], 'ea',
subject))
# check if subject has already been processed
if check_complete(ea_dir, subject):
msg = '{} already analysed'.format(subject)
logger.info(msg)
sys.exit(0)
# reset / remove error.log
error_log = os.path.join(output_dir, 'error.log')
if os.path.isfile(error_log):
os.remove(error_log)
# find the behavioural data, and exit if we fail to find it
try:
resdirs = glob.glob(os.path.join(resources_dir, subject + '_??'))
resources = []
for resdir in resdirs:
resfiles = [
os.path.join(dp, f) for dp, dn, fn in os.walk(resdir)
for f in fn
]
resources.extend(resfiles)
logs = filter(lambda x: '.log' in x and 'UCLAEmpAcc' in x, resources)
logs.sort()
except:
logger.error('No BEHAV data for {}.'.format(subject))
sys.exit(1)
# if we have the wrong number of logs, don't guess which to use, just fail
if len(logs) != 3:
error_message = 'Did not find exactly 3 logs for {}\nfound:{}.'.format(
subject, logs)
logger.error(error_message)
with open(error_log, 'wb') as f:
f.write('{}\n{}'.format(error_message, NODE))
sys.exit(1)
# parse and write the logs seperately for each experiment condition (video or shapes/colours video)
for test_type in ['vid', 'cvid']:
# extract all of the data from the logs
on_all, dur_all, corr_all, push_all, timings_all = [], [], [], [], []
try:
logger.info('Parsing {} logfiles for subject'.format(
len(logs), subject))
for log in logs:
# extract the block id from the logfilename
block_id = os.path.splitext(os.path.basename(log))[0][-1]
on, dur, corr, push, timings = process_behav_data(
log, output_dir, subject, test_type, block_id)
on_all.extend(on)
dur_all.extend(dur)
corr_all.extend(corr)
push_all.extend(push)
timings_all.extend(timings)
except Exception, e:
msg = 'Failed to parse logs for {}, with {}.'.format(
subject, str(e))
logger.error(msg)
sys.exit(1)
# write data to stimulus timing file for AFNI, and a QC csv
# on_all = sorted(on_all, key=lambda x:x[1])
timings_all = sorted(
timings_all, key=lambda x:
(x[2], x[3], x[1])) # put the responses into order
try:
logger.info('Writing stimulus data')
# write each stimulus time:
# [start_time]*[amplitude],[buttonpushes]:[block_length]
# 30*5,0.002:12
# OFFSET 4 TRs == 8 Seconds!
# on = on - 8.0
f1 = open('{}/{}_{}_block-times_ea.1D'.format(
output_dir, subject, test_type), 'wb') # stim timing file
f2 = open('{}/{}_{}_corr_push.csv'.format(output_dir, subject,
test_type),
'wb') # r values and num pushes / minute
f3 = open('{}/{}_{}_button-times.csv'.format(
output_dir, subject, test_type),
'wb') # button responses and timings
f4 = open('{}/{}_{}_vid-onsets.csv'.format(output_dir, subject,
test_type),
'wb') # button responses and timings
f2.write('correlation,n-pushes-per-minute\n')
f3.write('Block_ID,Video,Response,Timing\n')
f4.write('Block_ID,Video, Onset\n')
for i in range(len(on_all)):
f1.write('{o:.2f}*{r:.2f},{p}:{d:.2f} '.format(o=on_all[i][1] -
8.0,
r=corr_all[i],
p=push_all[i],
d=dur_all[i]))
f2.write('{r:.2f},{p}\n'.format(r=corr_all[i], p=push_all[i]))
for timing in timings_all:
f3.write('{b},{v},{r},{t:.2f}\n'.format(b=timing[2],
v=timing[3],
r=timing[0],
t=timing[1]))
for onset in on_all:
f4.write('{b},{r},{t:.2f}\n'.format(b=onset[2],
r=onset[0],
t=onset[1]))
f1.write(
'\n') # add newline at the end of each run (up to 3 runs.)
except IOError as e:
msg = 'Failed to open block_times & corr_push for {} with excuse {}'.format(
subject, e.strerror)
logger.error(msg)
finally:
f1.close()
f2.close()
f3.close()
f4.close()
