def get_new_subjects(config, qc_subjects): fs_subjects = [] for subject in qc_subjects: if sid.is_phantom(subject): logger.debug("Subject {} is a phantom. Skipping.".format(subject)) continue freesurfer_dir = utils.define_folder(config.get_path('freesurfer')) fs_subject_dir = os.path.join(freesurfer_dir, subject) if not outputs_exist(fs_subject_dir): fs_subjects.append(subject) return fs_subjects
def get_new_subjects(config, qc_subjects): fs_subjects = [] for subject in qc_subjects: if sid.is_phantom(subject): logger.debug("Subject {} is a phantom. Skipping.".format(subject)) continue freesurfer_dir = utils.define_folder(config.get_path('freesurfer')) fs_subject_dir = os.path.join(freesurfer_dir, subject) if not outputs_exist(fs_subject_dir): fs_subjects.append(subject) return fs_subjects
def main(): quit = "n" username = os.environ["XNAT_USER"] password = os.environ["XNAT_PASS"] central = Interface(server="https://xnat.imaging-genetics.camh.ca", user=username, password=password) while (quit != "y"): study = raw_input("Which study do you want to track scans for? ") con = CON.config() try: projects = set(con.get_xnat_projects(study)) except ValueError: print "Study does not exist" return 0 tracking_table = dict() for project in projects: constraints = [('xnat:mrSessionData/PROJECT', '=', project)] table = central.select('xnat:mrSessionData', [ 'xnat:mrSessionData/SUBJECT_LABEL', 'xnat:mrSessionData/DATE', 'xnat:mrSessionData/INSERT_DATE' ]).where(constraints) sort = sorted(table.items(), key=operator.itemgetter(2)) for item in sort: #print(item) site_name = scanid.parse(item[0]).site if scanid.is_phantom(item[0]): site_name += "_PHA" if "FBN" in item[0]: site_name += "_FBN" elif "ADN" in item[0]: site_name += "_ADN" site_dict = tracking_table.setdefault(site_name, dict()) last_update = site_dict.setdefault(uploaddate, datetime.min) current_update = datetime.strptime(item[2], datetimeformat) if last_update < current_update: site_dict[date] = item[1] site_dict[uploaddate] = current_update if last_update == datetime.min: site_dict[uploaddiff] = "No Other Uploads" else: site_dict[uploaddiff] = dttostr(current_update - last_update) #break printdict(tracking_table) quit = raw_input("Quit? y/n ")
def get_halted_subjects(fs_path, subjects): timed_out_msg = fs_scraper.FSLog._TIMEDOUT.format("") halted = [] for subject in subjects: if sid.is_phantom(subject): continue fs_folder = os.path.join(fs_path, subject) try: log = fs_scraper.FSLog(fs_folder) except: pass else: if log.status.startswith(timed_out_msg): halted.append(subject) return halted
def get_halted_subjects(fs_path, subjects): timed_out_msg = fs_scraper.FSLog._TIMEDOUT.format("") halted = [] for subject in subjects: if sid.is_phantom(subject): continue fs_folder = os.path.join(fs_path, subject) try: log = fs_scraper.FSLog(fs_folder) except: pass else: if log.status.startswith(timed_out_msg): halted.append(subject) return halted
def get_files(session, filename): """ Starts with a file in the nii folder Checks if the file is a DTI type, and session is not a phantom Checks to see if a SlicerTractography file exists in the dtiprep folder Returns a tuple(dti_file, tract_file) or none """ if not filename.endswith('.nii.gz'): logger.info('File:{} is not a nifti file. Skipping' .format(filename)) return try: ident, tag, series, desc = scanid.parse_filename(filename) except scanid.ParseException: logger.debug('Invalid filename:{}'.format(filename)) return if scanid.is_phantom(ident.get_full_subjectid_with_timepoint()): msg = "Session:{} is a phantom. Skipping".format(session) logger.info(msg) return if not tag in TAGS: msg = ("File:{} is not in taglist:{}. Skipping" .format(os.path.basename(filename), TAGS)) return base_name = scanid.make_filename(ident, tag, series, desc) + '_SlicerTractography.vtk' tract_file = os.path.join(DTIPREP_PATH, session, base_name) if not os.path.isfile(tract_file): logger.info('Tract file:{} not found.'.format(tract_file)) return return(filename, tract_file)
def get_files(session, filename): """ Starts with a file in the nii folder Checks if the file is a DTI type, and session is not a phantom Checks to see if a SlicerTractography file exists in the dtiprep folder Returns a tuple(dti_file, tract_file) or none """ if not filename.endswith('.nii.gz'): logger.info('File:{} is not a nifti file. Skipping' .format(filename)) return try: ident, tag, series, desc = scanid.parse_filename(filename) except scanid.ParseException: logger.debug('Invalid filename:{}'.format(filename)) return if scanid.is_phantom(ident.get_full_subjectid_with_timepoint()): msg = "Session:{} is a phantom. Skipping".format(session) logger.info(msg) return if not tag in TAGS: msg = ("File:{} is not in taglist:{}. Skipping" .format(os.path.basename(filename), TAGS)) return base_name = scanid.make_filename(ident, tag, series, desc) + '_SlicerTractography.vtk' tract_file = os.path.join(DTIPREP_PATH, session, base_name) if not os.path.isfile(tract_file): logger.info('Tract file:{} not found.'.format(tract_file)) return return(filename, tract_file)
def main(): """ Runs fmri data through the specified epitome script. """ arguments = docopt(__doc__) study = arguments['<study>'] scanid = arguments['--subject'] debug = arguments['--debug'] dryrun = arguments['--dry-run'] # configure logging logging.info('Starting') if debug: logger.setLevel(logging.DEBUG) # load config for study try: config = cfg.config(study=study) except ValueError: logger.error('study {} not defined'.format(study)) sys.exit(1) study_base = config.get_study_base(study) for k in ['nii', 'fmri', 'hcp']: if k not in config.site_config['paths']: logger.error("paths:{} not defined in site config".format(k)) sys.exit(1) for x in config.study_config['fmri'].iteritems(): for k in ['dims', 'del', 'pipeline', 'tags', 'export', 'tr']: if k not in x[1].keys(): logger.error("fmri:{}:{} not defined in configuration file".format(x[0], k)) sys.exit(1) nii_dir = os.path.join(study_base, config.site_config['paths']['nii']) if scanid: path = os.path.join(nii_dir, scanid) if '_PHA_' in scanid: sys.exit('Subject {} if a phantom, cannot be analyzed'.format(scanid)) try: run_epitome(path, config, study) except Exception as e: logging.error(e) sys.exit(1) # run in batch mode else: subjects = [] nii_dirs = glob.glob('{}/*'.format(nii_dir)) # find subjects where at least one expected output does not exist for path in nii_dirs: subject = os.path.basename(path) if sid.is_phantom(subject): logger.debug("Subject {} is a phantom. Skipping.".format(subject)) continue fmri_dir = utils.define_folder(os.path.join(study_base, config.site_config['paths']['fmri'])) for exp in config.study_config['fmri'].keys(): expected_names = config.study_config['fmri'][exp]['export'] subj_dir = os.path.join(fmri_dir, exp, subject) if not outputs_exist(subj_dir, expected_names): subjects.append(subject) break subjects = list(set(subjects)) # submit a list of calls to ourself, one per subject commands = [] if debug: debugopt = '--debug' else: debugopt = '' for subject in subjects: commands.append(" ".join(['python ', __file__, study, '--subject {} '.format(subject), debugopt])) if commands: logger.debug('queueing up the following commands:\n'+'\n'.join(commands)) for i, cmd in enumerate(commands): jobname = 'dm_fmri_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S")) jobfile = '/tmp/{}'.format(jobname) logfile = '/tmp/{}.log'.format(jobname) errfile = '/tmp/{}.err'.format(jobname) with open(jobfile, 'wb') as fid: fid.write('#!/bin/bash\n') fid.write(cmd) rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format( logfile, errfile, jobname, jobfile)) if rtn: logger.error("Job submission failed. Output follows.") logger.error("stdout: {}".format(out)) sys.exit(1)
def main(): """ Runs fmri data through the specified epitome script. """ arguments = docopt(__doc__) study = arguments['<study>'] scanid = arguments['--subject'] debug = arguments['--debug'] dryrun = arguments['--dry-run'] output = arguments['--output'] exports = arguments['--exports'] task = arguments['--task'] # configure logging logging.info('Starting') if debug: logger.setLevel(logging.DEBUG) # load config for study try: config = cfg.config(study=study) except ValueError: logger.error('study {} not defined'.format(study)) sys.exit(1) study_base = config.get_study_base(study) #Parse optional arguments output_dir = output if output else os.path.join(study_base,config.get_path('fmri')) opt_exports = [e for e in exports.split(',')] if exports else [] #Check if task is available if task: try: config.study_config['fmri'][task] except KeyError: logger.error('Task {} not found in study config!'.format(task)) sys.exit(1) tasks = {k:v for k,v in config.study_config['fmri'].iteritems() if k == task} else: tasks = config.study_config['fmri'] for k in ['nii', 'fmri', 'hcp']: if k not in config.get_key('Paths'): logger.error("paths:{} not defined in site config".format(k)) sys.exit(1) for x in tasks.iteritems(): for k in ['dims', 'del', 'pipeline', 'tags', 'export', 'tr']: if k not in x[1].keys(): logger.error("fmri:{}:{} not defined in configuration file".format(x[0], k)) sys.exit(1) nii_dir = os.path.join(study_base, config.get_path('nii')) if scanid: path = os.path.join(nii_dir, scanid) if '_PHA_' in scanid: sys.exit('Subject {} if a phantom, cannot be analyzed'.format(scanid)) try: run_epitome(path, config, study, output_dir, opt_exports,tasks) except Exception as e: logging.error(e) sys.exit(1) # run in batch mode else: subjects = [] nii_dirs = glob.glob('{}/*'.format(nii_dir)) # find subjects where at least one expected output does not exist for path in nii_dirs: subject = os.path.basename(path) if sid.is_phantom(subject): logger.debug("Subject {} is a phantom. Skipping.".format(subject)) continue fmri_dir = utils.define_folder(output_dir) for exp in config.study_config['fmri'].keys(): expected_names = set(config.study_config['fmri'][exp]['export'] + opt_exports) subj_dir = os.path.join(fmri_dir, exp, subject) if not outputs_exist(subj_dir, expected_names): subjects.append(subject) break subjects = list(set(subjects)) # submit a list of calls to ourself, one per subject commands = [] g_opts = ' --output {} --exports {}'.format(output_dir,exports) if task: g_opts += ' --task {}'.format(task) if debug: g_opts += ' --debug' for subject in subjects: sub_tag = ' --subject {}'.format(subject) commands.append(" ".join(['python ', __file__, study,g_opts,sub_tag])) if commands: logger.debug('queueing up the following commands:\n'+'\n'.join(commands)) for i, cmd in enumerate(commands): jobname = 'dm_fmri_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S")) jobfile = '/tmp/{}'.format(jobname) logfile = '/tmp/{}.log'.format(jobname) errfile = '/tmp/{}.err'.format(jobname) with open(jobfile, 'wb') as fid: fid.write('#!/bin/bash\n') fid.write(cmd) rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format( logfile, errfile, jobname, jobfile)) if rtn: logger.error("Job submission failed. Output follows.") logger.error("stdout: {}".format(out)) sys.exit(1)
if not sub_ids: sub_ids = os.listdir(nii_dir) sub_ids = sorted(sub_ids) #Run multi-submission only if multiple subjects with queue option enabled if (len(sub_ids) > 1) and (queue): for sub_id in sub_ids: logger.info('Submitting subject to queue: {}'.format(sub_id)) submit_dm_to_bids(log_dir, sub_id, arguments, cfg) #Run if either queue disabled or single subject else: for sub_id in sub_ids: subject_dir = sub_id if scanid.is_phantom(subject_dir): logger.info("File is phantom and will be ignored: {}".format( subject_dir)) sys.exit(1) parsed = scanid.parse(subject_dir) if os.path.isdir( os.path.join(bids_dir, to_sub(parsed), to_ses(parsed.timepoint))) and not rewrite: logger.warning( 'BIDS subject directory already exists. Exiting: {}'. format(subject_dir)) sys.exit(1) type_folders = create_bids_dirs(bids_dir, parsed) sub_nii_dir = os.path.join(nii_dir, subject_dir) + '/' logger.info(
def main(): arguments = docopt(__doc__) study = arguments['<study>'] use_server = arguments['--log-to-server'] debug = arguments['--debug'] config = load_config(study) if use_server: add_server_handler(config) if debug: logger.setLevel(logging.DEBUG) ## setup some paths study_base_dir = config.get_study_base() fs_dir = config.get_path('freesurfer') data_dir = config.get_path('nii') # not sure where to put this. Potentially it could be very large # keeping it means existing subjects don't get re-run. # it could be deleted but then would need extra code to Determine # if subjects have been run. working_dir = os.path.join(study_base_dir, 'pipelines/workingdir_reconflow') ## These are overrides, for testing base_dir = '/external/rprshnas01/tigrlab/' fs_dir = os.path.join(base_dir, 'scratch/twright/pipelines/freesurfer', study) working_dir = os.path.join( base_dir, 'scratch/twright/pipelines/workingdir_reconflow') # freesurfer fails if the subjects dir doesn't exist check_folder_exists(fs_dir) # get the list of subjects that are not phantoms and have been qc'd subject_list = config.get_subject_metadata() subject_list = [ subject for subject in subject_list if not dm_scanid.is_phantom(subject) ] # Need to determine if the study has T2 (or FLAIR) scans, # do this by looking in the study_config.yml for expected scantypes. # Current pipelines add T2 files if they exist on a per-subject basis # Nipype expects the each run of the pipeline to be the same across all subjects # it is possible to set some parameters on a per-subject basis (see nu-iter setting) # but is this desirable? scan_types = get_common_scan_types(config) if not 'T1' in scan_types: msg = 'Study {} does not have T1 scans, aborting.'.format(study) sys.exit(msg) templates = {'T1': '{dm_subject_id}/{dm_subject_id}_??_T1_??*.nii.gz'} if 'T2' in scan_types: templates['T2'] = '{dm_subject_id}/{dm_subject_id}_??_T2_??*.nii.gz' if 'FLAIR' in scan_types: logger.debug('FLAIR processing not yet implemented') #templates = {'T2': '{dm_subject_id}/{dm_subject_id}_??_FLAIR _??*.nii.gz'} # setup the nipype nodes # infosource justs iterates through the list of subjects infosource = Node(IdentityInterface(fields=['subject_id']), name="infosource") # For testing subject_list = ['DTI_CMH_H001_02'] infosource.iterables = ('subject_id', subject_list) # sf finds the files for each subject. The dmSelectFiles class # overrides the nipype.SelectFiles adding checks that the numbers # of files matches those defined in study_config.yml sf = Node(dmSelectFiles(templates), name="selectFiles") sf.inputs.base_directory = data_dir # set_nuiter implements a simple function to set the iteration count # on a subject by subject basis set_nuiter = Node(Function(input_names=['subject_id'], output_names=['nu_iter'], function=get_nuiter_settings), name='get_nuiter') # reconall is the interface for the recon-all freesurfer function # currently seem unable to specify multiple directives # (e.g. -qcache and -notal-check) reconall = Node(ReconAll(directive='all', parallel=True, subjects_dir=fs_dir), name='recon-all') # if this is running on a cluster, we can specify node specific requirements # i.e. reconall runs well with lots of cores. reconall.plugin_args = { 'qsub_args': '-l nodes=1:ppn=24', 'overwrite': True } # get_summary extracts the summary information from the output of reconall get_summary = Node(EnigmaSummaryTask(), name='Enigma_Summaries') ## Create the workflow reconflow = Workflow(name='reconflow') reconflow.base_dir = working_dir # need a different connection pattern and param for the reconall node # if T2 files exist sf_ra_conx = [('T1', 'T1_files')] if 'T2' in scan_types: reconall.inputs.use_T2 = True sf_ra_conx.append('T2', 'T2_file') ## Connect the outputs from each node to the corresponding inputs # Basically we link the defined outputs from each node, to the inputs of the next node # Each item in the list is [node1, node2, [(output_node1, input_node2)]] # Problem here due to incompatibilities between freesurfer 5 & 6 # this pattern works for freesurfer 5.3.0 (without the parallel flag for reconall) # but failes for 6.0.0, which doesn't support the nuierations flag. # reconflow.connect([(infosource, sf, [('subject_id', 'dm_subject_id')]), # (infosource, set_nuiter, [('subject_id', 'subject_id')]), # (sf, reconall, sf_ra_conx), # (set_nuiter, reconall, [('nu_iter', 'flags')])]) # this is the freesurfer 6 compatible version reconflow.connect([(infosource, sf, [('subject_id', 'dm_subject_id')]), (infosource, reconall, [('subject_id', 'subject_id')]), (sf, reconall, sf_ra_conx), (reconall, get_summary, [('subjects_dir', 'subjects_dir'), ('subject_id', 'subject_id'), ('subjects_dir', 'output_path')])]) # need to use a job template to ensure the environment is set correctly # on the running nodes. # Not sure why the current env isn't being passed job_template = os.path.join(os.path.dirname(__file__), 'job_template_scc.sh') ## run the actual workflow. # the pbsGraph plugin creates jobs for each node on a PBS torque using # torque scheduling to keep them in order. # Use plugin='SGEGraph' to run on lab cluster (not sure what will happen # to the reconflow node if we don't have any 24 core machines). # Don't specify a plugin to run on a single machine reconflow.run(plugin='PBSGraph', plugin_args=dict(template=job_template))