def get_new_subjects(config, qc_subjects):
fs_subjects = []
for subject in qc_subjects:
if sid.is_phantom(subject):
logger.debug("Subject {} is a phantom. Skipping.".format(subject))
continue
freesurfer_dir = utils.define_folder(config.get_path('freesurfer'))
fs_subject_dir = os.path.join(freesurfer_dir, subject)
if not outputs_exist(fs_subject_dir):
fs_subjects.append(subject)
return fs_subjects
def get_new_subjects(config, qc_subjects):
fs_subjects = []
for subject in qc_subjects:
if sid.is_phantom(subject):
logger.debug("Subject {} is a phantom. Skipping.".format(subject))
continue
freesurfer_dir = utils.define_folder(config.get_path('freesurfer'))
fs_subject_dir = os.path.join(freesurfer_dir, subject)
if not outputs_exist(fs_subject_dir):
fs_subjects.append(subject)
return fs_subjects
def main():
quit = "n"
username = os.environ["XNAT_USER"]
password = os.environ["XNAT_PASS"]
central = Interface(server="https://xnat.imaging-genetics.camh.ca",
user=username,
password=password)
while (quit != "y"):
study = raw_input("Which study do you want to track scans for? ")
con = CON.config()
try:
projects = set(con.get_xnat_projects(study))
except ValueError:
print "Study does not exist"
return 0
tracking_table = dict()
for project in projects:
constraints = [('xnat:mrSessionData/PROJECT', '=', project)]
table = central.select('xnat:mrSessionData', [
'xnat:mrSessionData/SUBJECT_LABEL', 'xnat:mrSessionData/DATE',
'xnat:mrSessionData/INSERT_DATE'
]).where(constraints)
sort = sorted(table.items(), key=operator.itemgetter(2))
for item in sort:
#print(item)
site_name = scanid.parse(item[0]).site
if scanid.is_phantom(item[0]):
site_name += "_PHA"
if "FBN" in item[0]:
site_name += "_FBN"
elif "ADN" in item[0]:
site_name += "_ADN"
site_dict = tracking_table.setdefault(site_name, dict())
last_update = site_dict.setdefault(uploaddate, datetime.min)
current_update = datetime.strptime(item[2], datetimeformat)
if last_update < current_update:
site_dict[date] = item[1]
site_dict[uploaddate] = current_update
if last_update == datetime.min:
site_dict[uploaddiff] = "No Other Uploads"
else:
site_dict[uploaddiff] = dttostr(current_update -
last_update)
#break
printdict(tracking_table)
quit = raw_input("Quit? y/n ")
def get_halted_subjects(fs_path, subjects):
timed_out_msg = fs_scraper.FSLog._TIMEDOUT.format("")
halted = []
for subject in subjects:
if sid.is_phantom(subject):
continue
fs_folder = os.path.join(fs_path, subject)
try:
log = fs_scraper.FSLog(fs_folder)
except:
pass
else:
if log.status.startswith(timed_out_msg):
halted.append(subject)
return halted
def get_halted_subjects(fs_path, subjects):
timed_out_msg = fs_scraper.FSLog._TIMEDOUT.format("")
halted = []
for subject in subjects:
if sid.is_phantom(subject):
continue
fs_folder = os.path.join(fs_path, subject)
try:
log = fs_scraper.FSLog(fs_folder)
except:
pass
else:
if log.status.startswith(timed_out_msg):
halted.append(subject)
return halted
def get_files(session, filename):
"""
Starts with a file in the nii folder
Checks if the file is a DTI type, and session is not a phantom
Checks to see if a SlicerTractography file exists in the dtiprep folder
Returns a tuple(dti_file, tract_file) or none
"""
if not filename.endswith('.nii.gz'):
logger.info('File:{} is not a nifti file. Skipping'
.format(filename))
return
try:
ident, tag, series, desc = scanid.parse_filename(filename)
except scanid.ParseException:
logger.debug('Invalid filename:{}'.format(filename))
return
if scanid.is_phantom(ident.get_full_subjectid_with_timepoint()):
msg = "Session:{} is a phantom. Skipping".format(session)
logger.info(msg)
return
if not tag in TAGS:
msg = ("File:{} is not in taglist:{}. Skipping"
.format(os.path.basename(filename),
TAGS))
return
base_name = scanid.make_filename(ident, tag, series, desc) + '_SlicerTractography.vtk'
tract_file = os.path.join(DTIPREP_PATH, session, base_name)
if not os.path.isfile(tract_file):
logger.info('Tract file:{} not found.'.format(tract_file))
return
return(filename, tract_file)
def get_files(session, filename):
"""
Starts with a file in the nii folder
Checks if the file is a DTI type, and session is not a phantom
Checks to see if a SlicerTractography file exists in the dtiprep folder
Returns a tuple(dti_file, tract_file) or none
"""
if not filename.endswith('.nii.gz'):
logger.info('File:{} is not a nifti file. Skipping'
.format(filename))
return
try:
ident, tag, series, desc = scanid.parse_filename(filename)
except scanid.ParseException:
logger.debug('Invalid filename:{}'.format(filename))
return
if scanid.is_phantom(ident.get_full_subjectid_with_timepoint()):
msg = "Session:{} is a phantom. Skipping".format(session)
logger.info(msg)
return
if not tag in TAGS:
msg = ("File:{} is not in taglist:{}. Skipping"
.format(os.path.basename(filename),
TAGS))
return
base_name = scanid.make_filename(ident, tag, series, desc) + '_SlicerTractography.vtk'
tract_file = os.path.join(DTIPREP_PATH, session, base_name)
if not os.path.isfile(tract_file):
logger.info('Tract file:{} not found.'.format(tract_file))
return
return(filename, tract_file)
def main():
"""
Runs fmri data through the specified epitome script.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['nii', 'fmri', 'hcp']:
if k not in config.site_config['paths']:
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
for x in config.study_config['fmri'].iteritems():
for k in ['dims', 'del', 'pipeline', 'tags', 'export', 'tr']:
if k not in x[1].keys():
logger.error("fmri:{}:{} not defined in configuration file".format(x[0], k))
sys.exit(1)
nii_dir = os.path.join(study_base, config.site_config['paths']['nii'])
if scanid:
path = os.path.join(nii_dir, scanid)
if '_PHA_' in scanid:
sys.exit('Subject {} if a phantom, cannot be analyzed'.format(scanid))
try:
run_epitome(path, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
# run in batch mode
else:
subjects = []
nii_dirs = glob.glob('{}/*'.format(nii_dir))
# find subjects where at least one expected output does not exist
for path in nii_dirs:
subject = os.path.basename(path)
if sid.is_phantom(subject):
logger.debug("Subject {} is a phantom. Skipping.".format(subject))
continue
fmri_dir = utils.define_folder(os.path.join(study_base, config.site_config['paths']['fmri']))
for exp in config.study_config['fmri'].keys():
expected_names = config.study_config['fmri'][exp]['export']
subj_dir = os.path.join(fmri_dir, exp, subject)
if not outputs_exist(subj_dir, expected_names):
subjects.append(subject)
break
subjects = list(set(subjects))
# submit a list of calls to ourself, one per subject
commands = []
if debug:
debugopt = '--debug'
else:
debugopt = ''
for subject in subjects:
commands.append(" ".join(['python ', __file__, study, '--subject {} '.format(subject), debugopt]))
if commands:
logger.debug('queueing up the following commands:\n'+'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_fmri_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def main():
"""
Runs fmri data through the specified epitome script.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
output = arguments['--output']
exports = arguments['--exports']
task = arguments['--task']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
#Parse optional arguments
output_dir = output if output else os.path.join(study_base,config.get_path('fmri'))
opt_exports = [e for e in exports.split(',')] if exports else []
#Check if task is available
if task:
try:
config.study_config['fmri'][task]
except KeyError:
logger.error('Task {} not found in study config!'.format(task))
sys.exit(1)
tasks = {k:v for k,v in config.study_config['fmri'].iteritems() if k == task}
else:
tasks = config.study_config['fmri']
for k in ['nii', 'fmri', 'hcp']:
if k not in config.get_key('Paths'):
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
for x in tasks.iteritems():
for k in ['dims', 'del', 'pipeline', 'tags', 'export', 'tr']:
if k not in x[1].keys():
logger.error("fmri:{}:{} not defined in configuration file".format(x[0], k))
sys.exit(1)
nii_dir = os.path.join(study_base, config.get_path('nii'))
if scanid:
path = os.path.join(nii_dir, scanid)
if '_PHA_' in scanid:
sys.exit('Subject {} if a phantom, cannot be analyzed'.format(scanid))
try:
run_epitome(path, config, study, output_dir, opt_exports,tasks)
except Exception as e:
logging.error(e)
sys.exit(1)
# run in batch mode
else:
subjects = []
nii_dirs = glob.glob('{}/*'.format(nii_dir))
# find subjects where at least one expected output does not exist
for path in nii_dirs:
subject = os.path.basename(path)
if sid.is_phantom(subject):
logger.debug("Subject {} is a phantom. Skipping.".format(subject))
continue
fmri_dir = utils.define_folder(output_dir)
for exp in config.study_config['fmri'].keys():
expected_names = set(config.study_config['fmri'][exp]['export'] + opt_exports)
subj_dir = os.path.join(fmri_dir, exp, subject)
if not outputs_exist(subj_dir, expected_names):
subjects.append(subject)
break
subjects = list(set(subjects))
# submit a list of calls to ourself, one per subject
commands = []
g_opts = ' --output {} --exports {}'.format(output_dir,exports)
if task:
g_opts += ' --task {}'.format(task)
if debug:
g_opts += ' --debug'
for subject in subjects:
sub_tag = ' --subject {}'.format(subject)
commands.append(" ".join(['python ', __file__, study,g_opts,sub_tag]))
if commands:
logger.debug('queueing up the following commands:\n'+'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_fmri_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
if not sub_ids:
sub_ids = os.listdir(nii_dir)
sub_ids = sorted(sub_ids)
#Run multi-submission only if multiple subjects with queue option enabled
if (len(sub_ids) > 1) and (queue):
for sub_id in sub_ids:
logger.info('Submitting subject to queue: {}'.format(sub_id))
submit_dm_to_bids(log_dir, sub_id, arguments, cfg)
#Run if either queue disabled or single subject
else:
for sub_id in sub_ids:
subject_dir = sub_id
if scanid.is_phantom(subject_dir):
logger.info("File is phantom and will be ignored: {}".format(
subject_dir))
sys.exit(1)
parsed = scanid.parse(subject_dir)
if os.path.isdir(
os.path.join(bids_dir, to_sub(parsed),
to_ses(parsed.timepoint))) and not rewrite:
logger.warning(
'BIDS subject directory already exists. Exiting: {}'.
format(subject_dir))
sys.exit(1)
type_folders = create_bids_dirs(bids_dir, parsed)
sub_nii_dir = os.path.join(nii_dir, subject_dir) + '/'
logger.info(
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
use_server = arguments['--log-to-server']
debug = arguments['--debug']
config = load_config(study)
if use_server:
add_server_handler(config)
if debug:
logger.setLevel(logging.DEBUG)
## setup some paths
study_base_dir = config.get_study_base()
fs_dir = config.get_path('freesurfer')
data_dir = config.get_path('nii')
# not sure where to put this. Potentially it could be very large
# keeping it means existing subjects don't get re-run.
# it could be deleted but then would need extra code to Determine
# if subjects have been run.
working_dir = os.path.join(study_base_dir,
'pipelines/workingdir_reconflow')
## These are overrides, for testing
base_dir = '/external/rprshnas01/tigrlab/'
fs_dir = os.path.join(base_dir, 'scratch/twright/pipelines/freesurfer',
study)
working_dir = os.path.join(
base_dir, 'scratch/twright/pipelines/workingdir_reconflow')
# freesurfer fails if the subjects dir doesn't exist
check_folder_exists(fs_dir)
# get the list of subjects that are not phantoms and have been qc'd
subject_list = config.get_subject_metadata()
subject_list = [
subject for subject in subject_list
if not dm_scanid.is_phantom(subject)
]
# Need to determine if the study has T2 (or FLAIR) scans,
# do this by looking in the study_config.yml for expected scantypes.
# Current pipelines add T2 files if they exist on a per-subject basis
# Nipype expects the each run of the pipeline to be the same across all subjects
# it is possible to set some parameters on a per-subject basis (see nu-iter setting)
# but is this desirable?
scan_types = get_common_scan_types(config)
if not 'T1' in scan_types:
msg = 'Study {} does not have T1 scans, aborting.'.format(study)
sys.exit(msg)
templates = {'T1': '{dm_subject_id}/{dm_subject_id}_??_T1_??*.nii.gz'}
if 'T2' in scan_types:
templates['T2'] = '{dm_subject_id}/{dm_subject_id}_??_T2_??*.nii.gz'
if 'FLAIR' in scan_types:
logger.debug('FLAIR processing not yet implemented')
#templates = {'T2': '{dm_subject_id}/{dm_subject_id}_??_FLAIR _??*.nii.gz'}
# setup the nipype nodes
# infosource justs iterates through the list of subjects
infosource = Node(IdentityInterface(fields=['subject_id']),
name="infosource")
# For testing
subject_list = ['DTI_CMH_H001_02']
infosource.iterables = ('subject_id', subject_list)
# sf finds the files for each subject. The dmSelectFiles class
# overrides the nipype.SelectFiles adding checks that the numbers
# of files matches those defined in study_config.yml
sf = Node(dmSelectFiles(templates), name="selectFiles")
sf.inputs.base_directory = data_dir
# set_nuiter implements a simple function to set the iteration count
# on a subject by subject basis
set_nuiter = Node(Function(input_names=['subject_id'],
output_names=['nu_iter'],
function=get_nuiter_settings),
name='get_nuiter')
# reconall is the interface for the recon-all freesurfer function
# currently seem unable to specify multiple directives
# (e.g. -qcache and -notal-check)
reconall = Node(ReconAll(directive='all',
parallel=True,
subjects_dir=fs_dir),
name='recon-all')
# if this is running on a cluster, we can specify node specific requirements
# i.e. reconall runs well with lots of cores.
reconall.plugin_args = {
'qsub_args': '-l nodes=1:ppn=24',
'overwrite': True
}
# get_summary extracts the summary information from the output of reconall
get_summary = Node(EnigmaSummaryTask(), name='Enigma_Summaries')
## Create the workflow
reconflow = Workflow(name='reconflow')
reconflow.base_dir = working_dir
# need a different connection pattern and param for the reconall node
# if T2 files exist
sf_ra_conx = [('T1', 'T1_files')]
if 'T2' in scan_types:
reconall.inputs.use_T2 = True
sf_ra_conx.append('T2', 'T2_file')
## Connect the outputs from each node to the corresponding inputs
# Basically we link the defined outputs from each node, to the inputs of the next node
# Each item in the list is [node1, node2, [(output_node1, input_node2)]]
# Problem here due to incompatibilities between freesurfer 5 & 6
# this pattern works for freesurfer 5.3.0 (without the parallel flag for reconall)
# but failes for 6.0.0, which doesn't support the nuierations flag.
# reconflow.connect([(infosource, sf, [('subject_id', 'dm_subject_id')]),
# (infosource, set_nuiter, [('subject_id', 'subject_id')]),
# (sf, reconall, sf_ra_conx),
# (set_nuiter, reconall, [('nu_iter', 'flags')])])
# this is the freesurfer 6 compatible version
reconflow.connect([(infosource, sf, [('subject_id', 'dm_subject_id')]),
(infosource, reconall, [('subject_id', 'subject_id')]),
(sf, reconall, sf_ra_conx),
(reconall, get_summary,
[('subjects_dir', 'subjects_dir'),
('subject_id', 'subject_id'),
('subjects_dir', 'output_path')])])
# need to use a job template to ensure the environment is set correctly
# on the running nodes.
# Not sure why the current env isn't being passed
job_template = os.path.join(os.path.dirname(__file__),
'job_template_scc.sh')
## run the actual workflow.
# the pbsGraph plugin creates jobs for each node on a PBS torque using
# torque scheduling to keep them in order.
# Use plugin='SGEGraph' to run on lab cluster (not sure what will happen
# to the reconflow node if we don't have any 24 core machines).
# Don't specify a plugin to run on a single machine
reconflow.run(plugin='PBSGraph', plugin_args=dict(template=job_template))
