def main(): arguments = docopt(__doc__) url = arguments['--URL'] study = arguments['<study>'] output_path = arguments['--output'] config = datman.config.config(study=study) meta_path = config.get_path('meta') token_file = 'ahrc_token' token_path = os.path.join(meta_path, token_file) output_path = os.path.join(config.get_path('data'), output_path) token = get_token(token_path) payload = get_payload(token) REDCap_variables = [ 'record_id', 'redcap_event_name', 'demo_sex_birth', 'demo_age_study_entry', 'demo_highest_grade_self', 'term_premature_yn' ] data = make_rest(url, payload, REDCap_variables) column_headers = [ 'record_id', 'group', 'sex', 'age', 'education', 'terminated' ] make_csv(output_path, data, column_headers)
def main(): global DRYRUN arguments = docopt(__doc__) project = arguments['<project>'] series = arguments['<series>'] blacklist = arguments['--blacklist'] ignored_paths = arguments['--ignore-path'] verbose = arguments['--verbose'] debug = arguments['--debug'] quiet = arguments['--quiet'] DRYRUN = arguments['--dry-run'] if verbose: logger.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) if quiet: logger.setLevel(logging.ERROR) config = datman.config.config(study=project) blacklist = get_blacklist(arguments['--blacklist'], series, config) logger.debug("Found blacklist data: {}".format(blacklist)) remove_blacklisted_items(blacklist, config, ignored_paths)
def main(): global DRYRUN arguments = docopt(__doc__) project = arguments['<project>'] xnat_cred = arguments['--xnat'] redcap_cred = arguments['--redcap'] site_config = arguments['--site-config'] verbose = arguments['--verbose'] debug = arguments['--debug'] quiet = arguments['--quiet'] DRYRUN = arguments['--dry-run'] # Set log format log_handler.setFormatter(logging.Formatter('[%(name)s] %(levelname)s - ' '{study}: %(message)s'.format(study=project))) if verbose: logger.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) if quiet: logger.setLevel(logging.ERROR) config = datman.config.config(filename=site_config, study=project) user_name, password = os.environ["XNAT_USER"], os.environ["XNAT_PASS"] xnat_url = get_xnat_url(config) scan_complete_records = get_project_redcap_records(config, redcap_cred) with datman.utils.XNATConnection(xnat_url, user_name, password) as connection: for record in scan_complete_records: link_shared_ids(config, connection, record)
def main(): arguments = docopt(__doc__) study = arguments['<study>'] session = arguments['<session>'] backupdir = arguments['--backupdir'] purgedb = arguments['--purgedb'] CFG.set_study(study) base_dir = CFG.get_study_base() logger.info('Searching folders:{}'.format(base_dir)) # Create the backup folder outdir = os.path.realpath(os.path.join(backupdir, session)) try: os.makedirs(outdir) except OSError: logger.error('Failed making backup directory:{}'.format(outdir)) return if not purge_filesystem(session, base_dir, outdir): # somethings gone wrong. End processing here. return if purgedb: try: db = datman.dashboard.dashboard(study) db.delete_session(session) except: return
def main(): global REWRITE arguments = docopt(__doc__) use_server = arguments['--log-to-server'] verbose = arguments['--verbose'] debug = arguments['--debug'] quiet = arguments['--quiet'] study = arguments['<study>'] session = arguments['<session>'] REWRITE = arguments['--rewrite'] config = get_config(study) if use_server: add_server_handler(config) if quiet: logger.setLevel(logging.ERROR) if verbose: logger.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) if session: subject = prepare_scan(session, config) qc_single_scan(subject, config) return qc_all_scans(config)
def main(): global DRYRUN, CONFIG arguments = docopt(__doc__) link_file = arguments['<link_file>'] src_session = arguments['<src_session>'] trg_session = arguments['<trg_session>'] tags = arguments['<tags>'] verbose = arguments['--verbose'] debug = arguments['--debug'] DRYRUN = arguments['--dry-run'] quiet = arguments['--quiet'] logger.setLevel(logging.WARN) if quiet: logger.setLevel(logging.ERROR) if verbose: logger.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) if link_file is not None: logger.info("Using link file {} to make links".format(link_file)) for line in read_link_file(link_file): link_session_data(line[0], line[1], line[2]) return create_linked_session(src_session, trg_session, tags)
def main(): from datman.docopt import docopt import sys arguments = docopt(__doc__) if arguments['--showheaders']: for archive in arguments['<archive>']: manifest = datman.utils.get_archive_headers(archive, stop_after_first=False) filepath, headers = manifest.items()[0] print ",".join([archive, filepath]) print "\t" + "\n\t".join(headers.dir()) return headers = arguments['--headers'] and arguments['--headers'].split(',') or \ default_headers[:] headers.insert(0, "Path") rows = [] for archive in arguments['<archive>']: manifest = datman.utils.get_archive_headers(archive) sortedseries = sorted(manifest.iteritems(), key=lambda x: x[1].get('SeriesNumber')) for path, dataset in sortedseries: row = dict([(header, dataset.get(header, "")) for header in headers]) row['Path'] = path rows.append(row) if arguments['--oneseries']: break data = pd.DataFrame(rows) print data.to_csv(index=False)
def main(): global DRYRUN, PARALLEL, LOG_DIR, SYSTEM arguments = docopt(__doc__) study = arguments['<study>'] use_server = arguments['--log-to-server'] scanid = arguments['--subject'] debug = arguments['--debug'] resubmit = arguments['--resubmit'] PARALLEL = arguments['--parallel'] DRYRUN = arguments['--dry-run'] # If you add an option/argument that needs to be propagated to subject jobs # after a batch submit make sure to add it to create_command() # If it needs to be propagated to recon-all add it to # get_freesurfer_arguments() config = load_config(study) if use_server: add_server_handler(config) if debug: logger.setLevel(logging.DEBUG) logger.info('Starting') check_input_paths(config) qc_subjects = config.get_subject_metadata() fs_path = config.get_path('freesurfer') LOG_DIR = make_error_log_dir(fs_path) SYSTEM = config.system if scanid: # single subject mode blacklisted_series = get_blacklist(qc_subjects, scanid) subject = dm_scan.Scan(scanid, config) if subject.is_phantom: sys.exit( 'Subject {} is a phantom, cannot be analyzed'.format(scanid)) run_freesurfer(subject, blacklisted_series, config, resubmit) return # batch mode update_aggregate_stats(config) destination = os.path.join(fs_path, 'freesurfer_aggregate_log.csv') update_aggregate_log(config, qc_subjects, destination) fs_subjects = get_new_subjects(config, qc_subjects) logger.info("Submitting {} new subjects".format(len(fs_subjects))) if resubmit: # Filter out subjects that were just submitted to reduce search space remaining_subjects = filter(lambda x: x not in fs_subjects, qc_subjects) halted_subjects = get_halted_subjects(fs_path, remaining_subjects) logger.info("Resubmitting {} subjects".format(len(halted_subjects))) fs_subjects.extend(halted_subjects) submit_proc_freesurfer(fs_path, fs_subjects, arguments)
def main(): arguments = docopt(__doc__) verbose = arguments['--verbose'] debug = arguments['--debug'] quiet = arguments['--quiet'] study = arguments['<study>'] session = arguments['<session>'] # setup logging logging.basicConfig() ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.WARN) logger.setLevel(logging.WARN) if quiet: logger.setLevel(logging.ERROR) ch.setLevel(logging.ERROR) if verbose: logger.setLevel(logging.INFO) ch.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - ' '%(levelname)s - %(message)s') ch.setFormatter(formatter) logger.addHandler(ch) # setup the config object cfg = datman.config.config(study=study) dir_nii = cfg.get_path('nii') dir_res = cfg.get_path('resources') # setup the dashboard object db = datman.dashboard.dashboard(study=study) if session: # single session defined on command line sessions = [session] else: # session not defined find all sessions in a project sessions = os.listdir(dir_res) logger.info('Processing {} sessions'.format(len(sessions))) for session in sessions: try: process_session(cfg, db, dir_nii, dir_res, session) except: logger.error('Failed processing session:{}'.format(session))
def main(): global cfg, DRYRUN arguments = docopt(__doc__) verbose = arguments['--verbose'] debug = arguments['--debug'] quiet = arguments['--quiet'] DRYRUN = arguments['--dry-run'] study = arguments['<study>'] session = arguments['<session>'] # setup logging ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.WARN) logger.setLevel(logging.WARN) if quiet: logger.setLevel(logging.ERROR) ch.setLevel(logging.ERROR) if verbose: logger.setLevel(logging.INFO) ch.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - ' '%(levelname)s - %(message)s') ch.setFormatter(formatter) logger.addHandler(ch) # setup the config object logger.info('Loading config') cfg = datman.config.config(study=study) nii_dir = cfg.get_path('nii') images = [] if session: base_dir = os.path.join(nii_dir, session) files = os.listdir(base_dir) add_session_PDT2s(files, images, base_dir) else: for root, dirs, files in os.walk(nii_dir): add_session_PDT2s(files, images, root) logger.info('Found {} splittable nifti files with tag "PDT2"'.format( len(images))) for image in images: split(image)
def main(): arguments = docopt(__doc__) x = abs(float(arguments['<x>'])) y = abs(float(arguments['<y>'])) z = abs(float(arguments['<z>'])) files = arguments['<files>'] if x == 0: x = None if y == 0: y = None if z == 0: z = None target = [x, y, z] # remove all input files with ORIG_RESOLUTION in the name files = filter(lambda x: 'ORIG_RESOLUTION' not in x, files) # loop through files, reslice as necessary for f in files: reslice(target, f)
def make_qc_command(subject_id, study): arguments = docopt(__doc__) use_server = arguments['--log-to-server'] verbose = arguments['--verbose'] debug = arguments['--debug'] quiet = arguments['--quiet'] command = " ".join([__file__, study, subject_id]) if verbose: command = " ".join([command, '-v']) if debug: command = " ".join([command, '-d']) if quiet: command = " ".join([command, '-q']) if use_server: command = " ".join([command, '--log-to-server']) if REWRITE: command = command + ' --rewrite' return command
def main(): arguments = docopt(__doc__) project = arguments['<project>'] output_loc = arguments['--output'] config_file = arguments['--config_file'] system = arguments['--system'] xnat_cred = arguments['--xnat-credentials'] quiet = arguments['--quiet'] debug = arguments['--debug'] verbose = arguments['--verbose'] if verbose: logger.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) if quiet: logger.setLevel(logging.ERROR) config = datman.config.config(filename=config_file, system=system, study=project) output_file = set_output_name(output_loc, config) xnat_url = get_xnat_url(config) username, password = datman.utils.get_xnat_credentials(config, xnat_cred) xnat_project_names = config.get_xnat_projects() logger.debug("Summarizing XNAT projects {}".format(xnat_project_names)) with datman.utils.XNATConnection(xnat_url, username, password) as xnat_connection: overviews = get_session_overviews(xnat_connection, xnat_project_names) with requests.Session() as session: session.auth = (username, password) MR_ids = get_MR_ids(session, xnat_url, xnat_project_names) merged_records = merge_overview_and_labels(overviews, MR_ids) write_overview_csv(merged_records, output_file)
def main(): global DRYRUN, CONFIG arguments = docopt(__doc__) link_file = arguments['<link_file>'] src_session = arguments['<src_session>'] trg_session = arguments['<trg_session>'] tags = arguments['<tags>'] verbose = arguments['--verbose'] debug = arguments['--debug'] DRYRUN = arguments['--dry-run'] quiet = arguments['--quiet'] logger.setLevel(logging.WARN) if quiet: logger.setLevel(logging.ERROR) if verbose: logger.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) if link_file is not None: logger.info("Using link file {} to make links".format(link_file)) for line in read_link_file(link_file): link_session_data(line[0], line[1], line[2]) return logger.info("Linking the provided source {} and target " \ "{}".format(src_session, trg_session)) tags = link_session_data(src_session, trg_session, tags) src_link_file = get_external_links_csv(src_session) trg_link_file = get_external_links_csv(trg_session) for link_file in [src_link_file, trg_link_file]: write_link_file(link_file, src_session, trg_session, tags)
def main(): global DRYRUN arguments = docopt(__doc__) study = arguments['<study>'] subject = arguments['<subject>'] use_server = arguments['--log-to-server'] debug = arguments['--debug'] DRYRUN = arguments['--dry-run'] config = datman.config.config(study=study) if use_server: add_server_handler(config) if debug: logging.getLogger().setLevel(logging.DEBUG) check_environment() if subject: run_pipeline(config, subject, arguments['<T1>'], arguments['<T2>']) return run_all_subjects(config, arguments)
continue make_job(f, out_path) def main(study, session=None): logger.info('Processing study:{}'.format(study)) if session: process_session(session) else: sessions = os.listdir(NII_PATH) logger.info('Found {} sessions.'.format(len(sessions))) for session in sessions: process_session(session) if __name__ == '__main__': arguments = docopt(__doc__) study = arguments['<study>'] session = arguments['<session>'] ATLAS_FILE = arguments['--atlas_file'] CLUSTER_DIR = arguments['--cluster_dir'] MRML_FILE = arguments['--mrml_file'] CLUSTER_PATTERN = arguments['--cluster-pattern'] CONTAINER = arguments['--mitk_container'] CLEANUP = arguments['--leave_temp_files'] LOGDIR = arguments['--logDir'] OVERWRITE = arguments['--rewrite'] TAGS = arguments['--tags'] TAGS = [tag.strip() for tag in TAGS.split(',')] QUIET = False
def main(): """ Runs fmri data through the specified epitome script. """ arguments = docopt(__doc__) study = arguments['<study>'] scanid = arguments['--subject'] debug = arguments['--debug'] dryrun = arguments['--dry-run'] # configure logging logging.info('Starting') if debug: logger.setLevel(logging.DEBUG) # load config for study try: config = cfg.config(study=study) except ValueError: logger.error('study {} not defined'.format(study)) sys.exit(1) study_base = config.get_study_base(study) for k in ['nii', 'fmri', 'hcp']: if k not in config.site_config['paths']: logger.error("paths:{} not defined in site config".format(k)) sys.exit(1) for x in config.study_config['fmri'].iteritems(): for k in ['dims', 'del', 'pipeline', 'tags', 'export', 'tr']: if k not in x[1].keys(): logger.error("fmri:{}:{} not defined in configuration file".format(x[0], k)) sys.exit(1) nii_dir = os.path.join(study_base, config.site_config['paths']['nii']) if scanid: path = os.path.join(nii_dir, scanid) if '_PHA_' in scanid: sys.exit('Subject {} if a phantom, cannot be analyzed'.format(scanid)) try: run_epitome(path, config, study) except Exception as e: logging.error(e) sys.exit(1) # run in batch mode else: subjects = [] nii_dirs = glob.glob('{}/*'.format(nii_dir)) # find subjects where at least one expected output does not exist for path in nii_dirs: subject = os.path.basename(path) if sid.is_phantom(subject): logger.debug("Subject {} is a phantom. Skipping.".format(subject)) continue fmri_dir = utils.define_folder(os.path.join(study_base, config.site_config['paths']['fmri'])) for exp in config.study_config['fmri'].keys(): expected_names = config.study_config['fmri'][exp]['export'] subj_dir = os.path.join(fmri_dir, exp, subject) if not outputs_exist(subj_dir, expected_names): subjects.append(subject) break subjects = list(set(subjects)) # submit a list of calls to ourself, one per subject commands = [] if debug: debugopt = '--debug' else: debugopt = '' for subject in subjects: commands.append(" ".join(['python ', __file__, study, '--subject {} '.format(subject), debugopt])) if commands: logger.debug('queueing up the following commands:\n'+'\n'.join(commands)) for i, cmd in enumerate(commands): jobname = 'dm_fmri_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S")) jobfile = '/tmp/{}'.format(jobname) logfile = '/tmp/{}.log'.format(jobname) errfile = '/tmp/{}.err'.format(jobname) with open(jobfile, 'wb') as fid: fid.write('#!/bin/bash\n') fid.write(cmd) rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format( logfile, errfile, jobname, jobfile)) if rtn: logger.error("Job submission failed. Output follows.") logger.error("stdout: {}".format(out)) sys.exit(1)
def main(): arguments = docopt(__doc__) study = arguments['<study>'] scanid = arguments['--subject'] debug = arguments['--debug'] dryrun = arguments['--dry-run'] # configure logging logging.info('Starting') if debug: logger.setLevel(logging.DEBUG) # load config for study try: config = cfg.config(study=study) except ValueError: logger.error('study {} not defined'.format(study)) sys.exit(1) study_base = config.get_study_base(study) for k in ['freesurfer', 'hcp']: if k not in config.site_config['paths']: logger.error("paths:{} not defined in site config".format(k)) sys.exit(1) freesurfer_dir = config.get_path('freesurfer') hcp_dir = config.get_path('hcp') if scanid: path = os.path.join(freesurfer_dir, scanid) try: run_hcp_convert(path, config, study) except Exception as e: logging.error(e) sys.exit(1) return config = cfg.config(study=study) qced_subjects = config.get_subject_metadata() # running for batch mode new_subjects = [] # find subjects where at least one expected output does not exist for subject in qced_subjects: subj_dir = os.path.join(hcp_dir, subject) if not ciftify_outputs_exist(subj_dir): if fs_outputs_exist(os.path.join(freesurfer_dir, subject)): new_subjects.append(subject) create_indices_bm(config, study) # submit a list of calls to ourself, one per subject commands = [] if debug: debugopt = '--debug' else: debugopt = '' for subject in new_subjects: commands.append(" ".join( [__file__, study, '--subject {} '.format(subject), debugopt])) if commands: logger.debug('queueing up the following commands:\n' + '\n'.join(commands)) for i, cmd in enumerate(commands): jobname = 'dm_fs2hcp_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S")) jobfile = '/tmp/{}'.format(jobname) logfile = '/tmp/{}.log'.format(jobname) errfile = '/tmp/{}.err'.format(jobname) with open(jobfile, 'wb') as fid: fid.write('#!/bin/bash\n') fid.write(cmd) rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format( logfile, errfile, jobname, jobfile)) if rtn: logger.error("Job submission failed. Output follows.") logger.error("stdout: {}".format(out)) sys.exit(1)
def main(): arguments = docopt(__doc__) study = arguments['<study>'] output_csv = arguments['<csv_file>'] verbose = arguments['--verbose'] debug = arguments['--debug'] quiet = arguments['--quiet'] if quiet: logger.setLevel(logging.ERROR) if verbose: logger.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) logging.info('Starting') # Check the yaml file can be read correctly logger.debug('Reading yaml file.') cfg = datman.config.config(study=study) # # ## Read in the configuration yaml file # if not os.path.isfile(config_yaml): # raise ValueError("configuration file {} not found. Try again." # .format(config_yaml)) # # ## load the yml file # with open(config_yaml, 'r') as stream: # CONFIG = yaml.load(stream) # # ## check that the required keys are there # ExpectedKeys = ['paths'] # diffs = set(ExpectedKeys) - set(CONFIG.keys()) # if len(diffs) > 0: # raise ImportError("configuration file missing {}".format(diffs)) #dcm_dir = CONFIG['paths']['dcm'] dcm_dir = cfg.get_path('dcm') logger.debug('Getting scan list for {}'.format(dcm_dir)) scans = datman.utils.get_folder_headers(dcm_dir) logger.info('Found {} scans'.format(len(scans))) headers = [ "FOLDER", "SUBJECT", "SESSION", "SCANDATE", "SITE", "SUBJECT/REPEAT/PHANTOM" ] results = [] for key, val in scans.iteritems(): res = process_scan(key, val) if res: result = [ key, res[0], res[1], datetime.strftime(res[3], '%Y-%m-%d'), res[2] ] if res[4]: result.append('PHANTOM') elif res[5]: result.append('REPEAT') else: result.append("SUBJECT") results.append(result) if output_csv: with open(output_csv, 'wb') as csvfile: csv_writer = csv.writer(csvfile) csv_writer.writerow(headers) for row in results: csv_writer.writerow(row) else: print(','.join(headers)) for row in results: print(','.join(row))
def main(): arguments = docopt(__doc__) study = arguments['<study>'] scanid = arguments['--subject'] debug = arguments['--debug'] logging.info('Starting') if debug: logger.setLevel(logging.DEBUG) # load config for study try: config = cfg.config(study=study) except ValueError: logger.error( 'study {} not defined in master configuration file\n{}'.format( study, NODE)) sys.exit(1) study_base = config.get_study_base(study) if 'fmri' not in config.site_config['paths']: logger.error( "paths:fmri not defined in site configuration file\n{}".format( NODE)) sys.exit(1) fmri_dir = os.path.join(study_base, config.site_config['paths']['fmri']) if scanid: path = os.path.join(fmri_dir, scanid) try: run_analysis(scanid, config, study) except Exception as e: logger.error(e) sys.exit(1) # run in batch mode else: # look for subjects with at least one fmri type missing outputs subjects = [] # loop through fmri experiments defined for exp in config.study_config['fmri'].keys(): expected_files = config.study_config['fmri'][exp]['conn'] fmri_dirs = glob.glob('{}/*'.format(os.path.join(fmri_dir, exp))) for subj_dir in fmri_dirs: candidates = glob.glob('{}/*'.format(subj_dir)) for filetype in expected_files: # add subject if outputs don't already exist if not filter( lambda x: '{}_roi-corrs.csv'.format(filetype) in x, candidates): subjects.append(os.path.basename(subj_dir)) break # collapse found subjects (do not double-count) and create a list of commands commands = [] subjects = list(set(subjects)) for subject in subjects: commands.append(" ".join( [__file__, study, '--subject {}'.format(subject)])) if commands: logger.debug('queueing up the following commands:\n' + '\n'.join(commands)) for i, cmd in enumerate(commands): jobname = 'dm_rest_{}_{}'.format( i, time.strftime("%Y%m%d-%H%M%S")) jobfile = '/tmp/{}'.format(jobname) logfile = '/tmp/{}.log'.format(jobname) errfile = '/tmp/{}.err'.format(jobname) with open(jobfile, 'wb') as fid: fid.write('#!/bin/bash\n') fid.write(cmd) rtn, out = utils.run( 'qsub -V -q main.q -o {} -e {} -N {} {}'.format( logfile, errfile, jobname, jobfile)) if rtn: logger.error( "Job submission failed. Output follows. {}".format( NODE)) logger.error("stdout: {}".format(out)) sys.exit(1)
def main(): arguments = docopt(__doc__) study = arguments['<study>'] use_server = arguments['--log-to-server'] debug = arguments['--debug'] config = load_config(study) if use_server: add_server_handler(config) if debug: logger.setLevel(logging.DEBUG) ## setup some paths study_base_dir = config.get_study_base() fs_dir = config.get_path('freesurfer') data_dir = config.get_path('nii') # not sure where to put this. Potentially it could be very large # keeping it means existing subjects don't get re-run. # it could be deleted but then would need extra code to Determine # if subjects have been run. working_dir = os.path.join(study_base_dir, 'pipelines/workingdir_reconflow') ## These are overrides, for testing base_dir = '/external/rprshnas01/tigrlab/' fs_dir = os.path.join(base_dir, 'scratch/twright/pipelines/freesurfer', study) working_dir = os.path.join( base_dir, 'scratch/twright/pipelines/workingdir_reconflow') # freesurfer fails if the subjects dir doesn't exist check_folder_exists(fs_dir) # get the list of subjects that are not phantoms and have been qc'd subject_list = config.get_subject_metadata() subject_list = [ subject for subject in subject_list if not dm_scanid.is_phantom(subject) ] # Need to determine if the study has T2 (or FLAIR) scans, # do this by looking in the study_config.yml for expected scantypes. # Current pipelines add T2 files if they exist on a per-subject basis # Nipype expects the each run of the pipeline to be the same across all subjects # it is possible to set some parameters on a per-subject basis (see nu-iter setting) # but is this desirable? scan_types = get_common_scan_types(config) if not 'T1' in scan_types: msg = 'Study {} does not have T1 scans, aborting.'.format(study) sys.exit(msg) templates = {'T1': '{dm_subject_id}/{dm_subject_id}_??_T1_??*.nii.gz'} if 'T2' in scan_types: templates['T2'] = '{dm_subject_id}/{dm_subject_id}_??_T2_??*.nii.gz' if 'FLAIR' in scan_types: logger.debug('FLAIR processing not yet implemented') #templates = {'T2': '{dm_subject_id}/{dm_subject_id}_??_FLAIR _??*.nii.gz'} # setup the nipype nodes # infosource justs iterates through the list of subjects infosource = Node(IdentityInterface(fields=['subject_id']), name="infosource") # For testing subject_list = ['DTI_CMH_H001_02'] infosource.iterables = ('subject_id', subject_list) # sf finds the files for each subject. The dmSelectFiles class # overrides the nipype.SelectFiles adding checks that the numbers # of files matches those defined in study_config.yml sf = Node(dmSelectFiles(templates), name="selectFiles") sf.inputs.base_directory = data_dir # set_nuiter implements a simple function to set the iteration count # on a subject by subject basis set_nuiter = Node(Function(input_names=['subject_id'], output_names=['nu_iter'], function=get_nuiter_settings), name='get_nuiter') # reconall is the interface for the recon-all freesurfer function # currently seem unable to specify multiple directives # (e.g. -qcache and -notal-check) reconall = Node(ReconAll(directive='all', parallel=True, subjects_dir=fs_dir), name='recon-all') # if this is running on a cluster, we can specify node specific requirements # i.e. reconall runs well with lots of cores. reconall.plugin_args = { 'qsub_args': '-l nodes=1:ppn=24', 'overwrite': True } # get_summary extracts the summary information from the output of reconall get_summary = Node(EnigmaSummaryTask(), name='Enigma_Summaries') ## Create the workflow reconflow = Workflow(name='reconflow') reconflow.base_dir = working_dir # need a different connection pattern and param for the reconall node # if T2 files exist sf_ra_conx = [('T1', 'T1_files')] if 'T2' in scan_types: reconall.inputs.use_T2 = True sf_ra_conx.append('T2', 'T2_file') ## Connect the outputs from each node to the corresponding inputs # Basically we link the defined outputs from each node, to the inputs of the next node # Each item in the list is [node1, node2, [(output_node1, input_node2)]] # Problem here due to incompatibilities between freesurfer 5 & 6 # this pattern works for freesurfer 5.3.0 (without the parallel flag for reconall) # but failes for 6.0.0, which doesn't support the nuierations flag. # reconflow.connect([(infosource, sf, [('subject_id', 'dm_subject_id')]), # (infosource, set_nuiter, [('subject_id', 'subject_id')]), # (sf, reconall, sf_ra_conx), # (set_nuiter, reconall, [('nu_iter', 'flags')])]) # this is the freesurfer 6 compatible version reconflow.connect([(infosource, sf, [('subject_id', 'dm_subject_id')]), (infosource, reconall, [('subject_id', 'subject_id')]), (sf, reconall, sf_ra_conx), (reconall, get_summary, [('subjects_dir', 'subjects_dir'), ('subject_id', 'subject_id'), ('subjects_dir', 'output_path')])]) # need to use a job template to ensure the environment is set correctly # on the running nodes. # Not sure why the current env isn't being passed job_template = os.path.join(os.path.dirname(__file__), 'job_template_scc.sh') ## run the actual workflow. # the pbsGraph plugin creates jobs for each node on a PBS torque using # torque scheduling to keep them in order. # Use plugin='SGEGraph' to run on lab cluster (not sure what will happen # to the reconflow node if we don't have any 24 core machines). # Don't specify a plugin to run on a single machine reconflow.run(plugin='PBSGraph', plugin_args=dict(template=job_template))
def main(): arguments = docopt(__doc__) study = arguments['<study>'] subject = arguments['--subject'] debug = arguments['--debug'] logging.info('Starting') if debug: logger.setLevel(logging.DEBUG) # load config for study try: config = cfg.config(study=study) except ValueError: logger.error('study {} not defined'.format(study)) sys.exit(1) study_base = config.get_study_base(study) if 'ea' not in config.study_config['fmri'].keys(): logger.error('ea not defined in fmri in {}'.format(config_file)) sys.exit(1) for k in ['nii', 'fmri', 'hcp']: if k not in config.site_config['paths']: logger.error("paths:{} not defined in {}".format(k, config_file)) sys.exit(1) ea_dir = os.path.join(study_base, config.site_config['paths']['fmri'], 'ea') nii_dir = os.path.join(study_base, config.site_config['paths']['nii']) if subject: if '_PHA_' in subject: logger.error("{} is a phantom, cannot analyze".format(subject)) sys.exit(1) analyze_subject(subject, config, study) else: # batch mode subjects = glob.glob('{}/*'.format(nii_dir)) commands = [] if debug: opts = '--debug' else: opts = '' for path in subjects: subject = os.path.basename(path) if check_complete(ea_dir, subject): logger.debug('{} already analysed'.format(subject)) else: commands.append(" ".join( [__file__, study, '--subject {}'.format(subject), opts])) if commands: logger.debug("queueing up the following commands:\n" + '\n'.join(commands)) for i, cmd in enumerate(commands): jobname = "dm_ea_{}_{}".format(i, time.strftime("%Y%m%d-%H%M%S")) jobfile = '/tmp/{}'.format(jobname) logfile = '/tmp/{}.log'.format(jobname) errfile = '/tmp/{}.err'.format(jobname) with open(jobfile, 'wb') as fid: fid.write('#!/bin/bash\n') fid.write(cmd) rtn, out = utils.run( 'qsub -V -q main.q -o {} -e {} -N {} {}'.format( logfile, errfile, jobname, jobfile)) # qbacth method -- might bring it back, but not needed #fd, path = tempfile.mkstemp() #os.write(fd, '\n'.join(commands)) #os.close(fd) #rtn, out, err = utils.run('qbatch -i --logdir {ld} -N {name} --walltime {wt} {cmds}'.format(ld=logdir, name=jobname, wt=walltime, cmds=path)) if rtn: logger.error( "Job submission failed\nstdout: {}".format(out)) sys.exit(1)
def main(): arguments = docopt(__doc__) study = arguments['<study>'] scanid = arguments['--subject'] debug = arguments['--debug'] dryrun = arguments['--dry-run'] # configure logging logging.info('Starting') if debug: logger.setLevel(logging.DEBUG) # load config for study try: config = cfg.config(study=study) except ValueError: logger.error('study {} not defined'.format(study)) sys.exit(1) study_base = config.get_study_base(study) for k in ['freesurfer', 'hcp']: if k not in config.site_config['paths']: logger.error("paths:{} not defined in site config".format(k)) sys.exit(1) freesurfer_dir = os.path.join(study_base, config.site_config['paths']['freesurfer']) hcp_dir = os.path.join(study_base, config.site_config['paths']['hcp']) if scanid: path = os.path.join(freesurfer_dir, scanid) try: run_hcp_convert(path, config, study) except Exception as e: logging.error(e) sys.exit(1) # run in batch mode else: subjects = [] freesurfer_dirs = glob.glob('{}/*'.format(freesurfer_dir)) # find subjects where at least one expected output does not exist for path in nii_dirs: subject = os.path.basename(path) hcp_dir = utils.define_folder(os.path.join(study_base, config.site_config['paths']['hcp'])) if not outputs_exist(subj_dir): subjects.append(subject) subjects = list(set(subjects)) # submit a list of calls to ourself, one per subject commands = [] if debug: debugopt = '--debug' else: debugopt = '' for subject in subjects: commands.append(" ".join([__file__, study, '--subject {} '.format(subject), debugopt])) if commands: logger.debug('queueing up the following commands:\n'+'\n'.join(commands)) for i, cmd in enumerate(commands): jobname = 'dm_fs2hcp_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S")) jobfile = '/tmp/{}'.format(jobname) logfile = '/tmp/{}.log'.format(jobname) errfile = '/tmp/{}.err'.format(jobname) with open(jobfile, 'wb') as fid: fid.write('#!/bin/bash\n') fid.write(cmd) rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format( logfile, errfile, jobname, jobfile)) #rtn, out = utils.run('echo bash -l {}/{} {} | qbatch -N {} --logdir {} --walltime {} -'.format(bin_dir, script, subid, jobname, logs_dir, walltime)) if rtn: logger.error("Job submission failed. Output follows.") logger.error("stdout: {}".format(out)) sys.exit(1)
def main(): """ Loops through subjects, preprocessing using supplied script, and runs a first-level GLM using AFNI (tent functions, 15 s window) on all subjects. """ arguments = docopt(__doc__) project = arguments['<project>'] tmp_path = arguments['<tmppath>'] script = arguments['<script>'] assets = arguments['<assets>'] verbose = arguments['--verbose'] debug = arguments['--debug'] if verbose: logger.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) data_path = dm.utils.define_folder(os.path.join(project, 'data')) nii_path = dm.utils.define_folder(os.path.join(data_path, 'nii')) func_path = dm.utils.define_folder(os.path.join(data_path, 'imob')) tmp_path = dm.utils.define_folder(tmp_path) _ = dm.utils.define_folder(os.path.join(project, 'logs')) log_path = dm.utils.define_folder(os.path.join(project, 'logs/imob')) list_of_names = [] tmpdict = {} subjects = dm.utils.get_subjects(nii_path) # preprocess for sub in subjects: if dm.scanid.is_phantom(sub) == True: logger.debug("Skipping phantom subject {}".format(sub)) continue if os.path.isfile(os.path.join(func_path, '{sub}/{sub}_preproc-complete.log'.format(sub=sub))) == True: continue try: logger.info("Preprocessing subject {}".format(sub)) name, tmpdict = process_functional_data(sub, data_path, log_path, tmp_path, tmpdict, script) list_of_names.append(name) except ValueError as ve: continue if len(list_of_names) > 0: dm.utils.run_dummy_q(list_of_names) # export for sub in tmpdict: if os.path.isfile(os.path.join(func_path, '{sub}/{sub}_preproc-complete.log'.format(sub=sub))) == True: continue try: logger.info("Exporting subject {}".format(sub)) export_data(sub, tmpdict[sub], func_path) except: logger.error('Failed to export {}'.format(sub)) continue else: continue # analyze for sub in subjects: if dm.scanid.is_phantom(sub) == True: continue if os.path.isfile(os.path.join(func_path, '{sub}/{sub}_analysis-complete.log'.format(sub=sub))) == True: continue try: logger.info("Analyzing subject {}".format(sub)) script = generate_analysis_script(sub, func_path, assets) if script: returncode, _, _ = dm.utils.run('bash {}'.format(script)) dm.utils.check_returncode(returncode) dm.utils.run('touch {func_path}/{sub}/{sub}_analysis-complete.log'.format(func_path=func_path, sub=sub)) except Exception, e: logger.exception('Failed to analyze IMOB data for {}.'.format(sub))
def main(): """ Runs .nrrd data through unring.py. """ arguments = docopt(__doc__) study = arguments['<study>'] batch = arguments['--batch'] debug = arguments['--debug'] dryrun = arguments['--dry-run'] # configure logging logging.info('Starting') if debug: logger.setLevel(logging.DEBUG) # load config for study try: config = cfg.config(study=study) except ValueError: logger.error('study {} not defined'.format(study)) sys.exit(1) study_base = config.get_study_base(study) for k in ['nrrd']: if k not in config.site_config['paths']: logger.error("paths:{} not defined in site config".format(k)) sys.exit(1) nrrd_dir = os.path.join(study_base, config.site_config['paths']['nrrd']) # runs in serial (due to MATLAB dependencies) if batch: try: run_all(nrrd_dir, config, study) except Exception as e: logging.error(e) sys.exit(1) # default behaviour: submit self to queue in batch mode else: if debug: debugopt = '--debug' else: debugopt = '' cmd = 'python {} {} --batch {}'.format(__file__, study, debugopt) jobname = 'dm_unring_{}'.format(time.strftime("%Y%m%d-%H%M%S")) jobfile = '/tmp/{}'.format(jobname) logfile = '/tmp/{}.log'.format(jobname) errfile = '/tmp/{}.err'.format(jobname) with open(jobfile, 'wb') as fid: fid.write('#!/bin/bash\n') fid.write(cmd) rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format( logfile, errfile, jobname, jobfile)) if rtn: logger.error("Job submission failed. Output follows.") logger.error("stdout: {}".format(out)) sys.exit(1)
def main(): global xnat global cfg global excluded_studies global DRYRUN global dashboard arguments = docopt(__doc__) verbose = arguments['--verbose'] debug = arguments['--debug'] quiet = arguments['--quiet'] study = arguments['<study>'] server = arguments['--server'] credfile = arguments['--credfile'] username = arguments['--username'] session = arguments['<session>'] db_ignore = arguments['--dont-update-dashboard'] if arguments['--dry-run']: DRYRUN = True db_ignore = True # setup logging ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.WARN) logger.setLevel(logging.WARN) if quiet: logger.setLevel(logging.ERROR) ch.setLevel(logging.ERROR) if verbose: logger.setLevel(logging.INFO) ch.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) ch.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s - %(name)s - {study} - ' '%(levelname)s - %(message)s'.format(study=study)) ch.setFormatter(formatter) logger.addHandler(ch) # setup the config object logger.info('Loading config') cfg = datman.config.config(study=study) # setup the xnat object if not server: try: server = 'https://{}:{}'.format(cfg.get_key(['XNATSERVER']), cfg.get_key(['XNATPORT'])) except KeyError: logger.error( 'Failed to get xnat server info for study:{}'.format(study)) return if username: password = getpass.getpass() else: #Moving away from storing credentials in text files """ if not credfile: credfile = os.path.join(cfg.get_path('meta', study), 'xnat-credentials') with open(credfile) as cf: lines = cf.readlines() username = lines[0].strip() password = lines[1].strip() """ username = os.environ["XNAT_USER"] password = os.environ["XNAT_PASS"] xnat = datman.xnat.xnat(server, username, password) # setup the dashboard object if not db_ignore: try: dashboard = datman.dashboard.dashboard(study) except datman.dashboard.DashboardException as e: logger.error('Failed to initialise dashboard') # get the list of xnat projects linked to the datman study xnat_projects = cfg.get_xnat_projects(study) sessions = [] if session: # if session has been provided on the command line, identify which # project it is in try: xnat_project = xnat.find_session(session, xnat_projects) except datman.exceptions.XnatException as e: raise e if not xnat_project: logger.error( 'Failed to find session:{} in xnat.' ' Ensure it is named correctly with timepoint and repeat.'. format(xnat_projects)) return sessions.append((xnat_project, session)) else: for project in xnat_projects: project_sessions = xnat.get_sessions(project) for session in project_sessions: try: i = datman.scanid.parse(session['label']) if not datman.scanid.is_phantom( session['label']) and i.session == '': # raise an exception if scan is not a phantom and series is missing raise datman.scanid.ParseException except datman.scanid.ParseException: logger.error( 'Invalid session id:{} in project:{}, skipping.'. format(session['label'], project)) continue sessions.append((project, session['label'])) logger.info('Found {} sessions for study: {}'.format(len(sessions), study)) for session in sessions: process_session(session)
def main(): """ Essentially, analyzes the resting-state data. 1) Runs functional data through a defined epitome script. 2) Extracts time series from the cortex using MRI-space ROIs. 3) Generates a correlation matrix for each subject. 4) Generates an experiment-wide correlation matrix. 5) Generates a set of graph metrics for each subject. """ arguments = docopt(__doc__) project = arguments['<project>'] script = arguments['<script>'] atlas = arguments['<atlas>'] subjects = arguments['<subject>'] tags = arguments['--tags'].split(',') checklistfile = arguments['--checklist'] verbose = arguments['--verbose'] debug = arguments['--debug'] if verbose: logger.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) # check inputs if not os.path.isfile(atlas): logger.error("Atlas {} does not exist".format(atlas)) sys.exit(-1) if not os.path.isfile(script): logger.error("Epitome script {} does not exist".format(script)) sys.exit(-1) # submit jobs if not working on single subject submit_mode = len(subjects) == 0 logger.debug("Subjects: {}".format(subjects)) logger.debug("Submit mode: {}".format(submit_mode)) nii_path = os.path.join(project, 'data', 'nii') subjects = subjects or dm.utils.get_subjects(nii_path) if checklistfile and not os.path.exists(checklistfile): logger.fatal('Checklist {} does not exist'.format(checklistfile)) sys.exit(1) logger.debug('Using checklist: {}'.format(checklistfile)) checklist = dm.checklist.load(checklistfile) for subject in subjects: if checklist.is_blacklisted(STAGE_NAME, subject): logger.info("Subject {} blacklisted. Skipping.".format(subject)) continue if is_complete(project, subject): logger.info("Subject {} processed. Skipping.".format(subject)) continue if dm.scanid.is_phantom(subject): logger.debug("Subject {} is a phantom. Skipping.".format(subject)) continue try: data = get_required_data(project, subject, tags, checklist) except MissingDataException, e: logger.error(e.message) continue if submit_mode: opts = '' opts += verbose and ' --verbose' or '' opts += debug and ' --debug' or '' opts += tags and ' --tags=' + ','.join(tags) or '' cmd = "{me} {opts} {project} {script} {atlas} {subject}".format( me=__file__, opts=opts, project=project, script=script, atlas=atlas, subject=subject) job_name = 'dm_rest_{}'.format(subject) memopts = 'h_vmem=3G,mem_free=3G,virtual_free=3G' stamp = time.strftime("%Y%m%d-%H%M%S") logfile = '{name}-{stamp}.log'.format(name=job_name, stamp=stamp) logpath = os.path.join(project, 'logs', 'rest', logfile) qsub = 'qsub -V -N {name} -l {memopts} -o {logpath} -j y -b y {cmd}'.format( name=job_name, memopts=memopts, logpath=logpath, cmd=cmd) logger.debug('exec: {}'.format(qsub)) dm.utils.run(qsub) else: process_subject(project, data, subject, tags, atlas, script)
def main(): # make the already_linked dict global as we are going to use it a lot global already_linked global lookup global DRYRUN arguments = docopt(__doc__) verbose = arguments['--verbose'] debug = arguments['--debug'] DRYRUN = arguments['--dry-run'] quiet = arguments['--quiet'] study = arguments['<study>'] lookup_path = arguments['--lookup'] scanid_field = arguments['--scanid-field'] zipfile = arguments['<zipfile>'] # setup logging ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.WARN) logger.setLevel(logging.WARN) if quiet: logger.setLevel(logging.ERROR) ch.setLevel(logging.ERROR) if verbose: logger.setLevel(logging.INFO) ch.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) ch.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s - %(name)s - {study} - ' '%(levelname)s - %(message)s'.format(study=study)) ch.setFormatter(formatter) logger.addHandler(ch) # setup the config object cfg = datman.config.config(study=study) if not lookup_path: lookup_path = os.path.join(cfg.get_path('meta'), 'scans.csv') dicom_path = cfg.get_path('dicom') zips_path = cfg.get_path('zips') if not os.path.isdir(dicom_path): logger.warning('Dicom path:{} doesnt exist'.format(dicom_path)) try: os.makedirs(dicom_path) except IOError: logger.error('Failed to create dicom path:{}'.format(dicom_path)) return if not os.path.isdir(zips_path): logger.error('Zips path:{} doesnt exist'.format(zips_path)) return try: lookup = pd.read_table(lookup_path, sep='\s+', dtype=str) except IOError: logger.error('Lookup file:{} not found'.format(lookup_path)) return # identify which zip files have already been linked already_linked = { os.path.realpath(f): f for f in glob.glob(os.path.join(dicom_path, '*')) if os.path.islink(f) } if zipfile: if isinstance(zipfile, basestring): zipfile = [zipfile] archives = [os.path.join(zips_path, zip) for zip in zipfile] else: archives = [ os.path.join(zips_path, archive) for archive in os.listdir(zips_path) if os.path.splitext(archive)[1] == '.zip' ] logger.info('Found {} archives'.format(len(archives))) for archive in archives: link_archive(archive, dicom_path, scanid_field)
def main(): """ 1) Runs functional data through a custom epitome script. 2) Extracts block onsets, durations, and parametric modulators from behavioual log files collected at the scanner (and stored in RESOURCES). 3) Writes out AFNI-formatted timing files as well as a GLM script per subject. 4) Executes this script, producing beta-weights for each subject. """ arguments = docopt(__doc__) project = arguments['<project>'] tmp_path = arguments['<tmppath>'] script = arguments['<script>'] assets = arguments['<assets>'] verbose = arguments['--verbose'] debug = arguments['--debug'] if verbose: logging.getLogger().setLevel(logging.INFO) if debug: logging.getLogger().setLevel(logging.DEBUG) data_path = dm.utils.define_folder(os.path.join(project, 'data')) nii_path = dm.utils.define_folder(os.path.join(data_path, 'nii')) func_path = dm.utils.define_folder(os.path.join(data_path, 'ea')) tmp_path = dm.utils.define_folder(tmp_path) _ = dm.utils.define_folder(os.path.join(project, 'logs')) log_path = dm.utils.define_folder(os.path.join(project, 'logs/ea')) list_of_names = [] tmpdict = {} # preprocess subjects = dm.utils.get_subjects(nii_path) for sub in subjects: if dm.scanid.is_phantom(sub) == True: continue if os.path.isfile(os.path.join(func_path, '{sub}/{sub}_preproc-complete.log'.format(sub=sub))) == True: continue try: name, tmpdict = process_functional_data(sub, data_path, log_path, tmp_path, tmpdict, script) list_of_names.append(name) except ValueError as ve: continue if len(list_of_names) > 0: dm.utils.run_dummy_q(list_of_names) # export for sub in tmpdict: if os.path.isfile(os.path.join(func_path, '{sub}/{sub}_preproc-complete.log'.format(sub=sub))) == True: continue try: export_data(sub, tmpdict[sub], func_path) except: logger.error('Failed to export {}'.format(sub)) continue else: continue # analyze subjects = dm.utils.get_subjects(func_path) for sub in subjects: if dm.scanid.is_phantom(sub) == True: continue if os.path.isfile('{func_path}/{sub}/{sub}_analysis-complete.log'.format(func_path=func_path, sub=sub)) == True: continue # get all the log files for a subject try: resdirs = glob.glob(os.path.join(data_path, 'RESOURCES', sub + '_??')) resources = [] for resdir in resdirs: resfiles = [os.path.join(dp, f) for dp, dn, fn in os.walk(resdir) for f in fn] resources.extend(resfiles) logs = filter(lambda x: '.log' in x and 'UCLAEmpAcc' in x, resources) logs.sort() except: logger.error('No BEHAV data for {}.'.format(sub)) continue if len(logs) != 3: logger.error('Did not find exactly 3 logs for {}.'.format(sub)) continue # exract all of the data from the logs on_all, dur_all, corr_all, push_all = [], [], [], [] try: for log in logs: on, dur, corr, push = process_behav_data(log, assets, func_path, sub, 'vid') on_all.extend(on) dur_all.extend(dur) corr_all.extend(corr) push_all.extend(push) except Exception, e: logger.error('Failed to parse logs for {}, log={}.'.format(sub, log)) continue # write data to stimulus timing file for AFNI, and a QC csv try: # write each stimulus time: # [start_time]*[amplitude],[buttonpushes]:[block_length] # 30*5,0.002:12 # OFFSET 4 TRs == 8 Seconds! # on = on - 8.0 f1 = open('{func_path}/{sub}/{sub}_block-times_ea.1D'.format(func_path=func_path, sub=sub), 'wb') # stim timing file f2 = open('{func_path}/{sub}/{sub}_corr_push.csv'.format(func_path=func_path, sub=sub), 'wb') # r values and num pushes / minute f2.write('correlation,n-pushes-per-minute\n') for i in range(len(on_all)): f1.write('{o:.2f}*{r:.2f},{p}:{d:.2f} '.format(o=on_all[i]-8.0, r=corr_all[i], p=push_all[i], d=dur_all[i])) f2.write('{r:.2f},{p}\n'.format(r=corr_all[i], p=push_all[i])) f1.write('\n') # add newline at the end of each run (up to 3 runs.) except: logger.error('Failed to open block_times & corr_push for {}'.format(sub)) continue finally: f1.close() f2.close() # analyze the data try: generate_analysis_script(sub, func_path) dm.utils.run('bash {func_path}/{sub}/{sub}_glm_1stlevel_cmd.sh'.format(func_path=func_path, sub=sub)) dm.utils.run('touch {func_path}/{sub}/{sub}_analysis-complete.log'.format(func_path=func_path, sub=sub)) except: continue
def main(): """ Loops through subjects, preprocessing using supplied script, and runs a first-level GLM using AFNI (tent functions, 15 s window) on all subjects. """ arguments = docopt(__doc__) study = arguments['<study>'] subject = arguments['--subject'] debug = arguments['--debug'] logging.info('Starting') if debug: logger.setLevel(logging.DEBUG) # load config for study try: config = cfg.config(study=study) except ValueError: logger.error('study {} not defined'.format(study)) sys.exit(1) study_base = config.get_study_base(study) imob_dir = os.path.join(study_base, config.site_config['paths']['fmri'], 'imob') # process a single subject if subject: # get required inputs from each files = glob.glob(os.path.join(imob_dir, subject) + '/*.nii.gz') inputs = get_inputs(files, config) # check if subject has already been processed if check_complete(imob_dir, subject): logger.info('{} already analysed'.format(subject)) sys.exit(0) # first level GLM for inputs for input_type in inputs.keys(): script = generate_analysis_script(subject, inputs, input_type, config, study) rtn, out = utils.run('chmod 754 {}'.format(script)) rtn, out = utils.run(script) if rtn: logger.error( 'Script {} failed to run on subject {} with error:\n{}'. format(script, subject, out)) sys.exit(1) # process all subjects else: commands = [] for path in glob.glob('{}/*'.format(imob_dir)): subject = os.path.basename(path) # add subject if any of the expected outputs do not exist files = glob.glob(os.path.join(imob_dir, subject) + '/*.nii.gz') try: inputs = get_inputs(files, config) except: logger.debug('Invalid inputs for {}'.format(subject)) continue expected = inputs.keys() for exp in expected: if not filter( lambda x: '{}_glm_IM_1stlvl_{}'.format(subject, exp) in x, files): commands.append(" ".join( [__file__, study, '--subject {}'.format(subject)])) break if commands: logger.debug("queueing up the following commands:\n" + '\n'.join(commands)) #fd, path = tempfile.mkstemp() #os.write(fd, '\n'.join(commands)) #os.close(fd) for i, cmd in enumerate(commands): jobname = "dm_imob_{}_{}".format( i, time.strftime("%Y%m%d-%H%M%S")) jobfile = '/tmp/{}'.format(jobname) logfile = '/tmp/{}.log'.format(jobname) errfile = '/tmp/{}.err'.format(jobname) with open(jobfile, 'wb') as fid: fid.write('#!/bin/bash\n') fid.write(cmd) rtn, out = utils.run( 'qsub -V -q main.q -o {} -e {} -N {} {}'.format( logfile, errfile, jobname, jobfile)) #rtn, out, err = utils.run('qbatch -i --logdir {logdir} -N {name} --walltime {wt} {cmds}'.format(logdir = log_path, name = jobname, wt = walltime, cmds = path)) if rtn: logger.error("Job submission failed. Output follows.") logger.error("stdout: {}".format(out)) sys.exit(1)
def main(): global username global server global password global XNAT global CFG arguments = docopt(__doc__) verbose = arguments['--verbose'] debug = arguments['--debug'] quiet = arguments['--quiet'] study = arguments['<study>'] server = arguments['--server'] credfile = arguments['--credfile'] username = arguments['--username'] archive = arguments['<archive>'] # setup logging ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.WARN) logger.setLevel(logging.WARN) if quiet: logger.setLevel(logging.ERROR) ch.setLevel(logging.ERROR) if verbose: logger.setLevel(logging.INFO) ch.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) ch.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s - %(name)s - {study} - ' '%(levelname)s - %(message)s'.format(study=study)) ch.setFormatter(formatter) logger.addHandler(ch) # setup the config object logger.info('Loading config') CFG = datman.config.config(study=study) XNAT = get_xnat(server=server, credfile=credfile, username=username) dicom_dir = CFG.get_path('dicom', study) # deal with a single archive specified on the command line, # otherwise process all files in dicom_dir if archive: # check if the specified archive is a valid file if os.path.isfile(archive): dicom_dir = os.path.dirname(os.path.normpath(archive)) archives = [os.path.basename(os.path.normpath(archive))] elif datman.scanid.is_scanid_with_session(archive): # a sessionid could have been provided, lets be nice and handle that archives = [datman.utils.splitext(archive)[0] + '.zip'] else: logger.error('Cant find archive:{}'.format(archive)) return else: archives = os.listdir(dicom_dir) logger.debug('Processing files in:{}'.format(dicom_dir)) logger.info('Processing {} files'.format(len(archives))) for archivefile in archives: process_archive(os.path.join(dicom_dir, archivefile))
def main(): arguments = docopt(__doc__) verbose = arguments['--verbose'] debug = arguments['--debug'] dryrun = arguments['--dry-run'] quiet = arguments['--quiet'] study = arguments['<study>'] # setup logging ch = logging.StreamHandler(sys.stdout) log_level = logging.WARN if quiet: log_level = logging.ERROR if verbose: log_level = logging.INFO if debug: log_level = logging.DEBUG logger.setLevel(log_level) ch.setLevel(log_level) logging.getLogger("paramiko").setLevel(log_level) formatter = logging.Formatter('%(asctime)s - %(name)s - {study} - '\ ' %(levelname)s - %(message)s'.format(study=study)) ch.setFormatter(formatter) logger.addHandler(ch) # setup the config object cfg = datman.config.config(study=study) # get folder information from the config object mrusers = cfg.get_key(['MRUSER']) mrfolders = cfg.get_key(['MRFOLDER']) mrserver = cfg.get_key(['FTPSERVER']) zips_path = cfg.get_path('zips') meta_path = cfg.get_path('meta') # Check the local project zips dir exists, create if not if not os.path.isdir(zips_path): logger.warning( 'Zips directory: {} not found; creating.'.format(zips_path)) if not dryrun: os.mkdir(zips_path) # MRfolders entry in config file should be a list, but could be a string if isinstance(mrfolders, basestring): mrfolders = [mrfolders] # MRUSER entry in config file should be a list, but could be a string if isinstance(mrusers, basestring): mrusers = [mrusers] # load the password pass_file = os.path.join(meta_path, 'mrftppass.txt') if not os.path.isfile(pass_file): logger.error('Password file: {} not found'.format(pass_file)) raise IOError passwords = [] with open(pass_file, 'r') as pass_file: for password in pass_file: password = password.strip() if password: passwords.append(password) # actually do the copying assert len(passwords) == len(mrusers), \ 'Each mruser in config should have and entry in the password file' for iloc in range(len(mrusers)): mruser = mrusers[iloc] password = passwords[iloc] with pysftp.Connection(mrserver, username=mruser, password=password) as sftp: valid_dirs = get_valid_remote_dirs(sftp, mrfolders) if len(valid_dirs) < 1: logger.error('Source folders:{} not found'.format(mrfolders)) for valid_dir in valid_dirs: # process each folder in turn logger.debug('Copying from:{} to:{}'.format( valid_dir, zips_path)) process_dir(sftp, valid_dir, zips_path)