def load_config(study):
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
return config
def main():
arguments = docopt(__doc__)
study = arguments["<study>"]
cfg = config.config(study=study)
subjects = arguments["--subject"]
bids_dir = arguments["--bids-dir"] or cfg.get_path("bids")
yml = arguments["--yaml"] or YAML
rewrite = arguments["--rewrite"]
debug = arguments["--debug"]
global ALLOW_INCOMPLETE
ALLOW_INCOMPLETE = arguments["--allow-incomplete"]
be = BIDSEnforcer(yml)
if debug:
logger.setLevel(logging.DEBUG)
make_dataset_description(bids_dir, study, be.version)
if not subjects:
subjects = os.listdir(cfg.get_path("nii"))
for s in subjects:
if "PHA" in s:
logger.info("{} is a Phantom scan - skipping...".format(s))
continue
logger.info("Processing: {}".format(s))
process_subject(s, cfg, be, bids_dir, rewrite)
def load_config(study):
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
return config
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
sub_ids = arguments['<sub-id>']
nii_dir = arguments['--nii-dir']
bids_dir = arguments['--bids-dir']
fmriprep_dir = arguments['--fmriprep-out-dir']
fs_dir = arguments['--freesurfer-dir']
rewrite = arguments['--rewrite']
to_server = arguments['--log-to-server']
debug = arguments['--debug']
queue = arguments['--use-queue']
cfg = config.config(study=study)
logger.info("Study to convert to BIDS Format: {}".format(study))
if not bids_dir:
bids_dir = os.path.join(cfg.get_path('data'), "bids/")
create_dir(bids_dir)
log_dir = os.path.join(bids_dir, 'logs')
create_dir(log_dir)
setup_logger(log_dir, to_server, debug, cfg, sub_ids)
logger.info("BIDS folder will be {}".format(bids_dir))
if not nii_dir:
nii_dir = cfg.get_path('nii')
logger.info(
"Nii files to be converted to BIDS format will be from: {}".format(
nii_dir))
if fmriprep_dir:
fmriprep_fs_dir = os.path.join(fmriprep_dir, 'freesurfer')
create_dir(fmriprep_fs_dir)
logger.info(
'Fmriprep freesurfer dir will be:{}'.format(fmriprep_fs_dir))
else:
fmriprep_fs_dir = None
if not fs_dir:
fs_dir = cfg.get_path('freesurfer')
logger.info('Freesurfer dir is: {}'.format(fs_dir))
all_tags = init_setup(study, cfg, bids_dir)
create_task_json(bids_dir, tag_map['fmri'])
to_delete = set()
try:
sites = cfg.get_sites()
except KeyError, err:
logger.error(err)
sys.exit(1)
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
path = arguments['<path>']
try:
cfg = config.config()
p = cfg.get_path(path, study=study)
print(p)
except Exception as e:
eprint(str(e))
def main():
config = cfg.config()
studies = config.get_key('Projects').keys()
for study in studies:
try:
config.set_study(study)
except Exception:
pass
is_open = config.get_key('IsOpen')
dashboard.set_study_status(study, is_open)
def main():
quit = "n"
username = os.environ["XNAT_USER"]
password = os.environ["XNAT_PASS"]
central = Interface(server="https://xnat.imaging-genetics.camh.ca",
user=username,
password=password)
while (quit != "y"):
study = raw_input("Which study do you want to track scans for? ")
con = CON.config()
try:
projects = set(con.get_xnat_projects(study))
except ValueError:
print "Study does not exist"
return 0
tracking_table = dict()
for project in projects:
constraints = [('xnat:mrSessionData/PROJECT', '=', project)]
table = central.select('xnat:mrSessionData', [
'xnat:mrSessionData/SUBJECT_LABEL', 'xnat:mrSessionData/DATE',
'xnat:mrSessionData/INSERT_DATE'
]).where(constraints)
sort = sorted(table.items(), key=operator.itemgetter(2))
for item in sort:
#print(item)
site_name = scanid.parse(item[0]).site
if scanid.is_phantom(item[0]):
site_name += "_PHA"
if "FBN" in item[0]:
site_name += "_FBN"
elif "ADN" in item[0]:
site_name += "_ADN"
site_dict = tracking_table.setdefault(site_name, dict())
last_update = site_dict.setdefault(uploaddate, datetime.min)
current_update = datetime.strptime(item[2], datetimeformat)
if last_update < current_update:
site_dict[date] = item[1]
site_dict[uploaddate] = current_update
if last_update == datetime.min:
site_dict[uploaddiff] = "No Other Uploads"
else:
site_dict[uploaddiff] = dttostr(current_update -
last_update)
#break
printdict(tracking_table)
quit = raw_input("Quit? y/n ")
def main():
arguments = docopt.docopt(__doc__)
rootdir = arguments['--root']
if arguments['--study']:
cfg = config.config()
rootdir = cfg.get_study_base(arguments['--study'])
for projectdir in get_project_dirs(rootdir):
checklist = os.path.join(projectdir, 'metadata', 'checklist.csv')
checklistdict = read_checklist(checklist)
for timepointdir in sorted(glob.glob(projectdir + '/data/nii/*')):
if '_PHA_' in timepointdir:
continue
timepoint = os.path.basename(timepointdir)
qcdocname = 'qc_' + timepoint
qcdoc = os.path.join(projectdir, 'qc', timepoint,
(qcdocname + '.html'))
data_mtime = max(
map(get_mtime,
glob.glob(timepointdir + '/*.nii.gz') + [timepointdir]))
# notify about missing QC reports or those with no checklist entry
if qcdocname not in checklistdict:
print('No checklist entry for {}'.format(timepointdir))
continue
elif not os.path.exists(qcdoc):
print('No QC doc generated for {}'.format(timepointdir))
continue
# find QC documents that are older than the most recent data export
if arguments['--show-newer'] and data_mtime > os.path.getmtime(
qcdoc):
newer = filter(
lambda x: os.path.getmtime(x) > os.path.getmtime(qcdoc),
glob.glob(timepointdir + '/*'))
if newer != []:
print('{}: QC doc is older than data in folder {} {} {}'.
format(qcdoc, timepointdir, data_mtime,
os.path.getmtime(qcdoc)))
print('\t' + '\n\t'.join(newer))
# notify about unchecked QC reports
if not checklistdict[qcdocname]:
print '{}: QC doc not signed off on'.format(qcdoc)
def get_nuiter_settings(subject_id):
"""
Returns the site specific nu_iter settings for the pipeline
Note as this is run as a function node I'm not sure how to handle logging
>>> get_nuiter_settings('SPN01_CMH_0001_01')
4
>>> get_nuiter_settings('SPN01_MRC_0001_01')
8
"""
import datman.config as cfg
import datman.scanid as scanid
default_value = '-nuiterations 4'
config = cfg.config()
ident = scanid.parse(subject_id)
site = ident.site
try:
study = config.map_xnat_archive_to_project(ident.study)
config.set_study(study)
except ValueError:
# logger.warning('Study:{} not defined in config'.format(study))
return (default_value)
try:
settings = config.get_key('freesurfer')
nu_iter_settings = settings['nu_iter']
except KeyError:
# logger.warning('Freesurfer setting not found')
return (default_value)
try:
if site in nu_iter_settings:
iter_count = nu_iter_settings[site]
elif 'DEFAULT' in nu_iter_settings:
iter_count = nu_iter_settings['DEFAULT']
except TypeError:
# incase the nu_iter isn't defined as a dict ()
iter_count = nu_iter_settings
return ('-nuiterations {}'.format(iter_count))
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
subject = arguments['--subject']
debug = arguments['--debug']
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
if 'ea' not in config.study_config['fmri'].keys():
logger.error('ea not defined in fmri in {}'.format(config_file))
sys.exit(1)
for k in ['nii', 'fmri', 'hcp']:
if k not in config.get_key('Paths'):
logger.error("paths:{} not defined in {}".format(k, config_file))
sys.exit(1)
ea_dir = os.path.join(study_base, config.get_path('fmri'), 'ea')
nii_dir = os.path.join(study_base, config.get_path('nii'))
if subject:
subjects = [subject]
else:
subjects = glob.glob('{}/*'.format(nii_dir))
for s in subjects:
if '_PHA_' in subject:
logger.error("{} if a phantom, cannot analyze".format(s))
continue
analyze_subject(s, config, study)
def main():
arguments = docopt(__doc__)
rootdir = arguments['--root']
if arguments['--study']:
cfg = config.config()
rootdir = cfg.get_study_base(arguments['--study'])
for projectdir in get_project_dirs(rootdir):
checklist = os.path.join(projectdir, 'metadata', 'checklist.csv')
checklistdict = datman.utils.read_checklist(path=checklist)
for timepointdir in sorted(glob.glob(os.path.join(projectdir, 'data',
'nii', '*'))):
if '_PHA_' in timepointdir:
continue
timepoint = os.path.basename(timepointdir)
qcdocname = 'qc_' + timepoint
qcdoc = os.path.join(projectdir, 'qc', timepoint, (qcdocname + '.html'))
data_mtime = max(map(get_mtime, glob.glob(timepointdir + '/*.nii.gz')+[timepointdir]))
# notify about missing QC reports or those with no checklist entry
if timepoint not in checklistdict:
print('No checklist entry for {}'.format(timepointdir))
continue
elif not os.path.exists(qcdoc):
print('No QC doc generated for {}'.format(timepointdir))
continue
# find QC documents that are older than the most recent data export
if arguments['--show-newer'] and data_mtime > os.path.getmtime(qcdoc):
newer = filter(lambda x: os.path.getmtime(x) > os.path.getmtime(qcdoc), glob.glob(timepointdir + '/*'))
if newer != []:
print('{}: QC doc is older than data in folder {} {} {}'.format(qcdoc, timepointdir, data_mtime, os.path.getmtime(qcdoc)))
print('\t' + '\n\t'.join(newer))
# notify about unchecked QC reports
if not checklistdict[timepoint]:
print '{}: QC doc not signed off on'.format(qcdoc)
def main():
"""
Runs fmri data through the specified epitome script.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['nii', 'fmri', 'hcp']:
if k not in config.site_config['paths']:
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
for x in config.study_config['fmri'].iteritems():
for k in ['dims', 'del', 'pipeline', 'tags', 'export', 'tr']:
if k not in x[1].keys():
logger.error("fmri:{}:{} not defined in configuration file".format(x[0], k))
sys.exit(1)
nii_dir = os.path.join(study_base, config.site_config['paths']['nii'])
if scanid:
path = os.path.join(nii_dir, scanid)
if '_PHA_' in scanid:
sys.exit('Subject {} if a phantom, cannot be analyzed'.format(scanid))
try:
run_epitome(path, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
# run in batch mode
else:
subjects = []
nii_dirs = glob.glob('{}/*'.format(nii_dir))
# find subjects where at least one expected output does not exist
for path in nii_dirs:
subject = os.path.basename(path)
if sid.is_phantom(subject):
logger.debug("Subject {} is a phantom. Skipping.".format(subject))
continue
fmri_dir = utils.define_folder(os.path.join(study_base, config.site_config['paths']['fmri']))
for exp in config.study_config['fmri'].keys():
expected_names = config.study_config['fmri'][exp]['export']
subj_dir = os.path.join(fmri_dir, exp, subject)
if not outputs_exist(subj_dir, expected_names):
subjects.append(subject)
break
subjects = list(set(subjects))
# submit a list of calls to ourself, one per subject
commands = []
if debug:
debugopt = '--debug'
else:
debugopt = ''
for subject in subjects:
commands.append(" ".join(['python ', __file__, study, '--subject {} '.format(subject), debugopt]))
if commands:
logger.debug('queueing up the following commands:\n'+'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_fmri_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def main():
"""
Runs .nrrd data through unring.py.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
batch = arguments['--batch']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['nrrd']:
if k not in config.get_path('Paths'):
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
nrrd_dir = os.path.join(study_base, config.get_path('nrrd'))
# runs in serial (due to MATLAB dependencies)
if batch:
try:
run_all(nrrd_dir, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
# default behaviour: submit self to queue in batch mode
else:
if debug:
debugopt = '--debug'
else:
debugopt = ''
cmd = 'python {} {} --batch {}'.format(__file__, study, debugopt)
jobname = 'dm_unring_{}'.format(time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run(
'qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error(
'study {} not defined in master configuration file\n{}'.format(
study, NODE))
sys.exit(1)
study_base = config.get_study_base(study)
if 'fmri' not in config.site_config['paths']:
logger.error(
"paths:fmri not defined in site configuration file\n{}".format(
NODE))
sys.exit(1)
fmri_dir = os.path.join(study_base, config.site_config['paths']['fmri'])
if scanid:
path = os.path.join(fmri_dir, scanid)
try:
run_analysis(scanid, config, study)
except Exception as e:
logger.error(e)
sys.exit(1)
# run in batch mode
else:
# look for subjects with at least one fmri type missing outputs
subjects = []
# loop through fmri experiments defined
for exp in config.study_config['fmri'].keys():
expected_files = config.study_config['fmri'][exp]['conn']
fmri_dirs = glob.glob('{}/*'.format(os.path.join(fmri_dir, exp)))
for subj_dir in fmri_dirs:
candidates = glob.glob('{}/*'.format(subj_dir))
for filetype in expected_files:
# add subject if outputs don't already exist
if not filter(
lambda x: '{}_roi-corrs.csv'.format(filetype) in x,
candidates):
subjects.append(os.path.basename(subj_dir))
break
# collapse found subjects (do not double-count) and create a list of commands
commands = []
subjects = list(set(subjects))
for subject in subjects:
commands.append(" ".join(
[__file__, study, '--subject {}'.format(subject)]))
if commands:
logger.debug('queueing up the following commands:\n' +
'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_rest_{}_{}'.format(
i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run(
'qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error(
"Job submission failed. Output follows. {}".format(
NODE))
logger.error("stdout: {}".format(out))
sys.exit(1)
def main():
"""
Runs fmri data through the specified epitome script.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
output = arguments['--output']
exports = arguments['--exports']
task = arguments['--task']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
#Parse optional arguments
output_dir = output if output else os.path.join(study_base,config.get_path('fmri'))
opt_exports = [e for e in exports.split(',')] if exports else []
#Check if task is available
if task:
try:
config.study_config['fmri'][task]
except KeyError:
logger.error('Task {} not found in study config!'.format(task))
sys.exit(1)
tasks = {k:v for k,v in config.study_config['fmri'].iteritems() if k == task}
else:
tasks = config.study_config['fmri']
for k in ['nii', 'fmri', 'hcp']:
if k not in config.get_key('Paths'):
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
for x in tasks.iteritems():
for k in ['dims', 'del', 'pipeline', 'tags', 'export', 'tr']:
if k not in x[1].keys():
logger.error("fmri:{}:{} not defined in configuration file".format(x[0], k))
sys.exit(1)
nii_dir = os.path.join(study_base, config.get_path('nii'))
if scanid:
path = os.path.join(nii_dir, scanid)
if '_PHA_' in scanid:
sys.exit('Subject {} if a phantom, cannot be analyzed'.format(scanid))
try:
run_epitome(path, config, study, output_dir, opt_exports,tasks)
except Exception as e:
logging.error(e)
sys.exit(1)
# run in batch mode
else:
subjects = []
nii_dirs = glob.glob('{}/*'.format(nii_dir))
# find subjects where at least one expected output does not exist
for path in nii_dirs:
subject = os.path.basename(path)
if sid.is_phantom(subject):
logger.debug("Subject {} is a phantom. Skipping.".format(subject))
continue
fmri_dir = utils.define_folder(output_dir)
for exp in config.study_config['fmri'].keys():
expected_names = set(config.study_config['fmri'][exp]['export'] + opt_exports)
subj_dir = os.path.join(fmri_dir, exp, subject)
if not outputs_exist(subj_dir, expected_names):
subjects.append(subject)
break
subjects = list(set(subjects))
# submit a list of calls to ourself, one per subject
commands = []
g_opts = ' --output {} --exports {}'.format(output_dir,exports)
if task:
g_opts += ' --task {}'.format(task)
if debug:
g_opts += ' --debug'
for subject in subjects:
sub_tag = ' --subject {}'.format(subject)
commands.append(" ".join(['python ', __file__, study,g_opts,sub_tag]))
if commands:
logger.debug('queueing up the following commands:\n'+'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_fmri_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
TAGS = arguments['--tags']
TAGS = [tag.strip() for tag in TAGS.split(',')]
QUIET = False
DEBUG = False
if arguments['--debug']:
DEBUG = True
logger.setLevel(logging.DEBUG)
elif arguments['--quiet']:
QUIET = True
logger.setLevel(logging.ERROR)
else:
logger.setLevel(logging.WARN)
CFG = config.config(study=study)
DTIPREP_PATH = CFG.get_path('dtiprep')
NII_PATH = CFG.get_path('nii')
if not LOGDIR:
LOGDIR = os.path.join(DTIPREP_PATH, 'tractmap_logs')
if not os.path.isdir(LOGDIR):
logger.info("Creating log dir:{}".format(LOGDIR))
try:
os.mkdir(LOGDIR)
except OSError:
msg = 'Failed creating log directory"{}'.format(LOGDIR)
logger.error(msg)
sys.exit(msg)
def test_initialise_from_environ():
os.environ['DM_CONFIG'] = os.path.join(FIXTURE_DIR, 'site_config.yml')
os.environ['DM_SYSTEM'] = 'test'
cfg = config.config()
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['freesurfer', 'hcp']:
if k not in config.get_key('Paths'):
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
freesurfer_dir = config.get_path('freesurfer')
hcp_dir = config.get_path('hcp')
logs_dir = make_error_log_dir(hcp_dir)
if scanid:
path = os.path.join(freesurfer_dir, scanid)
try:
run_hcp_convert(path, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
return
qced_subjects = utils.get_subject_metadata(config)
# running for batch mode
commands = []
# find subjects where at least one expected output does not exist
for subject in qced_subjects:
fs_outputs = os.path.join(freesurfer_dir, subject)
if not fs_outputs_exist(fs_outputs):
continue
subj_dir = os.path.join(hcp_dir, subject)
if not ciftify_outputs_exist(subj_dir):
cmd = make_subject_cmd(study, subject, debug=debug)
commands.append(cmd)
elif not qc_outputs_exist(subj_dir):
cmd = make_vis_cmd(hcp_dir, subject, debug=debug)
commands.append(cmd)
if commands:
logger.debug('queueing up the following commands:\n'+'\n'.join(commands))
# submit a list of calls to ourself, one per subject
for i, cmd in enumerate(commands):
job_name = 'dm_fs2hcp_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
utils.submit_job(cmd, job_name, logs_dir,
system = config.system, dryrun = dryrun)
create_indices_bm(config, study)
TAGS = arguments['--tags']
TAGS = [tag.strip() for tag in TAGS.split(',')]
QUIET = False
DEBUG = False
if arguments['--debug']:
DEBUG = True
logger.setLevel(logging.DEBUG)
elif arguments['--quiet']:
QUIET = True
logger.setLevel(logging.ERROR)
else:
logger.setLevel(logging.WARN)
CFG = config.config(study=study)
DTIPREP_PATH = CFG.get_path('dtiprep')
NII_PATH = CFG.get_path('nii')
if not LOGDIR:
LOGDIR = os.path.join(DTIPREP_PATH, 'tractmap_logs')
if not os.path.isdir(LOGDIR):
logger.info("Creating log dir:{}".format(LOGDIR))
try:
os.mkdir(LOGDIR)
except OSError:
msg = 'Failed creating log directory"{}'.format(LOGDIR)
logger.error(msg)
sys.exit(msg)
def test_initialise_from_environ():
os.environ['DM_CONFIG'] = os.path.join(FIXTURE_DIR, 'site_config.yml')
os.environ['DM_SYSTEM'] = 'test'
config.config()
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['freesurfer', 'hcp']:
if k not in config.site_config['paths']:
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
freesurfer_dir = config.get_path('freesurfer')
hcp_dir = config.get_path('hcp')
if scanid:
path = os.path.join(freesurfer_dir, scanid)
try:
run_hcp_convert(path, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
return
config = cfg.config(study=study)
qced_subjects = config.get_subject_metadata()
# running for batch mode
new_subjects = []
# find subjects where at least one expected output does not exist
for subject in qced_subjects:
subj_dir = os.path.join(hcp_dir, subject)
if not ciftify_outputs_exist(subj_dir):
if fs_outputs_exist(os.path.join(freesurfer_dir, subject)):
new_subjects.append(subject)
create_indices_bm(config, study)
# submit a list of calls to ourself, one per subject
commands = []
if debug:
debugopt = '--debug'
else:
debugopt = ''
for subject in new_subjects:
commands.append(" ".join(
[__file__, study, '--subject {} '.format(subject), debugopt]))
if commands:
logger.debug('queueing up the following commands:\n' +
'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_fs2hcp_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def main():
"""
Runs .nrrd data through unring.py.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
batch = arguments['--batch']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['nrrd']:
if k not in config.get_path('Paths'):
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
nrrd_dir = os.path.join(study_base, config.get_path('nrrd'))
# runs in serial (due to MATLAB dependencies)
if batch:
try:
run_all(nrrd_dir, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
# default behaviour: submit self to queue in batch mode
else:
if debug:
debugopt = '--debug'
else:
debugopt = ''
cmd = 'python {} {} --batch {}'.format(__file__, study, debugopt)
jobname = 'dm_unring_{}'.format(time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
subject = arguments['--subject']
debug = arguments['--debug']
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
if 'ea' not in config.study_config['fmri'].keys():
logger.error('ea not defined in fmri in {}'.format(config_file))
sys.exit(1)
for k in ['nii', 'fmri', 'hcp']:
if k not in config.site_config['paths']:
logger.error("paths:{} not defined in {}".format(k, config_file))
sys.exit(1)
ea_dir = os.path.join(study_base, config.site_config['paths']['fmri'],
'ea')
nii_dir = os.path.join(study_base, config.site_config['paths']['nii'])
if subject:
if '_PHA_' in subject:
logger.error("{} is a phantom, cannot analyze".format(subject))
sys.exit(1)
analyze_subject(subject, config, study)
else:
# batch mode
subjects = glob.glob('{}/*'.format(nii_dir))
commands = []
if debug:
opts = '--debug'
else:
opts = ''
for path in subjects:
subject = os.path.basename(path)
if check_complete(ea_dir, subject):
logger.debug('{} already analysed'.format(subject))
else:
commands.append(" ".join(
[__file__, study, '--subject {}'.format(subject), opts]))
if commands:
logger.debug("queueing up the following commands:\n" +
'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = "dm_ea_{}_{}".format(i,
time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run(
'qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
# qbacth method -- might bring it back, but not needed
#fd, path = tempfile.mkstemp()
#os.write(fd, '\n'.join(commands))
#os.close(fd)
#rtn, out, err = utils.run('qbatch -i --logdir {ld} -N {name} --walltime {wt} {cmds}'.format(ld=logdir, name=jobname, wt=walltime, cmds=path))
if rtn:
logger.error(
"Job submission failed\nstdout: {}".format(out))
sys.exit(1)
def main():
"""
Loops through subjects, preprocessing using supplied script, and runs a
first-level GLM using AFNI (tent functions, 15 s window) on all subjects.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
subject = arguments['--subject']
debug = arguments['--debug']
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
imob_dir = os.path.join(study_base, config.site_config['paths']['fmri'],
'imob')
# process a single subject
if subject:
# get required inputs from each
files = glob.glob(os.path.join(imob_dir, subject) + '/*.nii.gz')
inputs = get_inputs(files, config)
# check if subject has already been processed
if check_complete(imob_dir, subject):
logger.info('{} already analysed'.format(subject))
sys.exit(0)
# first level GLM for inputs
for input_type in inputs.keys():
script = generate_analysis_script(subject, inputs, input_type,
config, study)
rtn, out = utils.run('chmod 754 {}'.format(script))
rtn, out = utils.run(script)
if rtn:
logger.error(
'Script {} failed to run on subject {} with error:\n{}'.
format(script, subject, out))
sys.exit(1)
# process all subjects
else:
commands = []
for path in glob.glob('{}/*'.format(imob_dir)):
subject = os.path.basename(path)
# add subject if any of the expected outputs do not exist
files = glob.glob(os.path.join(imob_dir, subject) + '/*.nii.gz')
try:
inputs = get_inputs(files, config)
except:
logger.debug('Invalid inputs for {}'.format(subject))
continue
expected = inputs.keys()
for exp in expected:
if not filter(
lambda x: '{}_glm_IM_1stlvl_{}'.format(subject, exp) in
x, files):
commands.append(" ".join(
[__file__, study, '--subject {}'.format(subject)]))
break
if commands:
logger.debug("queueing up the following commands:\n" +
'\n'.join(commands))
#fd, path = tempfile.mkstemp()
#os.write(fd, '\n'.join(commands))
#os.close(fd)
for i, cmd in enumerate(commands):
jobname = "dm_imob_{}_{}".format(
i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run(
'qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
#rtn, out, err = utils.run('qbatch -i --logdir {logdir} -N {name} --walltime {wt} {cmds}'.format(logdir = log_path, name = jobname, wt = walltime, cmds = path))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
import datman.config as cfg
import datman.scan
# Necessary to silence all logging from dm_qc_report during tests.
logging.disable(logging.CRITICAL)
qc = importlib.import_module('bin.dm_qc_report')
FIXTURE = "tests/fixture_project_settings"
site_config_path = os.path.join(FIXTURE, 'site_config.yaml')
system = 'local'
study = 'STUDY'
config = cfg.config(filename=site_config_path, system=system, study=study)
class GetConfig(unittest.TestCase):
@nose.tools.raises(SystemExit)
def test_exits_gracefully_with_bad_study(self):
config = qc.get_config(study="madeupcode")
@nose.tools.raises(SystemExit)
@patch('datman.config.config')
def test_exits_gracefully_when_paths_missing_from_config(self, mock_config):
mock_config.return_value.get_path.side_effect = lambda path: {'dcm': '',
'nii': ''}[path]
config = qc.get_config("STUDY")
class VerifyInputPaths(unittest.TestCase):
def main():
"""
Loops through subjects, preprocessing using supplied script, and runs a
first-level GLM using AFNI (tent functions, 15 s window) on all subjects.
"""
arguments = docopt(__doc__)
study = arguments['<study>']
subject = arguments['--subject']
debug = arguments['--debug']
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
imob_dir = os.path.join(study_base, config.get_path('fmri'), 'imob')
# process a single subject
if subject:
# get required inputs from each
files = glob.glob(os.path.join(imob_dir, subject) + '/*.nii.gz')
inputs = get_inputs(files, config)
# check if subject has already been processed
if check_complete(imob_dir, subject):
logger.info('{} already analysed'.format(subject))
sys.exit(0)
# first level GLM for inputs
for input_type in inputs.keys():
script = generate_analysis_script(subject, inputs, input_type, config, study)
rtn, out = utils.run('chmod 754 {}'.format(script))
rtn, out = utils.run(script)
if rtn:
logger.error('Script {} failed to run on subject {} with error:\n{}'.format(
script, subject, out))
sys.exit(1)
# process all subjects
else:
commands = []
for path in glob.glob('{}/*'.format(imob_dir)):
subject = os.path.basename(path)
# add subject if any of the expected outputs do not exist
files = glob.glob(os.path.join(imob_dir, subject) + '/*.nii.gz')
try:
inputs = get_inputs(files, config)
except:
logger.debug('Invalid inputs for {}'.format(subject))
continue
expected = inputs.keys()
for exp in expected:
if not filter(lambda x: '{}_glm_IM_1stlvl_{}'.format(subject, exp) in x, files):
commands.append(" ".join([__file__, study, '--subject {}'.format(subject)]))
break
if commands:
logger.debug("queueing up the following commands:\n"+'\n'.join(commands))
#fd, path = tempfile.mkstemp()
#os.write(fd, '\n'.join(commands))
#os.close(fd)
for i, cmd in enumerate(commands):
jobname = "dm_imob_{}_{}".format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
#rtn, out, err = utils.run('qbatch -i --logdir {logdir} -N {name} --walltime {wt} {cmds}'.format(logdir = log_path, name = jobname, wt = walltime, cmds = path))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
def __init__(self, templates, **kwargs):
self.dm_config = dm_cfg.config()
super(dmSelectFiles, self).__init__(templates, **kwargs)
class TestScan(unittest.TestCase):
good_name = "STUDY_CMH_9999_01"
bad_name = "STUDYCMH_9999"
phantom = "STUDY_CMH_PHA_XXX9999"
config = cfg.config(filename=site_config, system=system, study=study)
def test_raises_parse_exception_with_bad_subject_id(self):
with pytest.raises(datman.scanid.ParseException):
datman.scan.Scan(self.bad_name, self.config)
def test_makes_scan_instance_for_id_without_session(self):
subject = datman.scan.Scan(self.good_name, self.config)
assert subject is not None
# Check that the missing session was set to the default
assert subject.session == '01'
def test_makes_scan_instance_for_phantom(self):
subject = datman.scan.Scan(self.phantom, self.config)
assert subject is not None
assert subject.full_id == self.phantom
def test_is_phantom_sets_correctly(self):
subject = datman.scan.Scan(self.good_name, self.config)
phantom = datman.scan.Scan(self.phantom, self.config)
assert not subject.is_phantom
assert phantom.is_phantom
@patch('os.path.exists')
def test_resources_paths_uses_full_id_plus_session(self, mock_exists):
mock_exists.return_value = True
subject = datman.scan.Scan(self.good_name, self.config)
expected_path = self.config.get_path('resources') + \
"STUDY_CMH_9999_01_01"
assert subject.resource_path == expected_path
def test_returns_expected_subject_paths(self):
subject = datman.scan.Scan(self.good_name, self.config)
expected_nii = self.config.get_path('nii') + self.good_name
expected_qc = self.config.get_path('qc') + self.good_name
assert subject.nii_path == expected_nii
assert subject.qc_path == expected_qc
def test_niftis_set_to_empty_list_when_broken_path(self):
subject = datman.scan.Scan(self.good_name, self.config)
assert subject.niftis == []
@patch('glob.glob')
def test_niftis_with_either_extension_type_found(self, mock_glob):
simple_ext = "{}_01_T1_02_SagT1-BRAVO.nii".format(self.good_name)
complex_ext = "{}_01_DTI60-1000_05_Ax-DTI-60.nii.gz".format(
self.good_name)
wrong_ext = "{}_01_DTI60-1000_05_Ax-DTI-60.bvec".format(self.good_name)
nii_list = [simple_ext, complex_ext, wrong_ext]
mock_glob.return_value = nii_list
subject = datman.scan.Scan(self.good_name, self.config)
found_niftis = [series.path for series in subject.niftis]
expected = [simple_ext, complex_ext]
assert sorted(found_niftis) == sorted(expected)
@patch('glob.glob')
def test_subject_series_with_nondatman_name_causes_parse_exception(
self,
mock_glob):
well_named = "{}_01_T1_02_SagT1-BRAVO.nii".format(self.good_name)
badly_named1 = "{}_01_DTI60-1000_05_Ax-DTI-60.nii".format(
self.bad_name)
badly_named2 = "{}_01_T2_07.nii".format(self.good_name)
nii_list = [well_named, badly_named1, badly_named2]
mock_glob.return_value = nii_list
with pytest.raises(datman.scanid.ParseException):
datman.scan.Scan(self.good_name, self.config)
@patch('glob.glob')
def test_nii_tags_lists_all_tags(self, mock_glob):
T1 = "STUDY_CAMH_9999_01_01_T1_02_SagT1-BRAVO.nii"
DTI = "STUDY_CAMH_9999_01_01_DTI60-1000_05_Ax-DTI-60.nii"
mock_glob.return_value = [T1, DTI]
subject = datman.scan.Scan(self.good_name, self.config)
assert sorted(subject.nii_tags) == sorted(['T1', 'DTI60-1000'])
@patch('glob.glob')
def test_get_tagged_nii_finds_all_matching_series(self, mock_glob):
T1_1 = "STUDY_CAMH_9999_01_01_T1_02_SagT1-BRAVO.nii"
T1_2 = "STUDY_CAMH_9999_01_01_T1_03_SagT1-BRAVO.nii.gz"
DTI = "STUDY_CAMH_9999_01_01_DTI_05_Ax-DTI-60.nii"
mock_glob.return_value = [T1_1, DTI, T1_2]
subject = datman.scan.Scan(self.good_name, self.config)
actual_T1s = [series.path for series in subject.get_tagged_nii('T1')]
expected = [T1_1, T1_2]
assert sorted(actual_T1s) == sorted(expected)
actual_DTIs = [series.path for series in subject.get_tagged_nii('DTI')]
expected = [DTI]
assert actual_DTIs == expected
@patch('glob.glob')
def test_get_tagged_X_returns_empty_list_when_no_tag_files(self,
mock_glob):
nifti = "STUDY_CAMH_9999_01_01_T1_03_SagT1-BRAVO.nii.gz"
mock_glob.return_value = [nifti]
subject = datman.scan.Scan(self.good_name, self.config)
assert subject.get_tagged_nii('DTI') == []
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
scanid = arguments['--subject']
debug = arguments['--debug']
dryrun = arguments['--dry-run']
# configure logging
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
for k in ['freesurfer', 'hcp']:
if k not in config.site_config['paths']:
logger.error("paths:{} not defined in site config".format(k))
sys.exit(1)
freesurfer_dir = os.path.join(study_base, config.site_config['paths']['freesurfer'])
hcp_dir = os.path.join(study_base, config.site_config['paths']['hcp'])
if scanid:
path = os.path.join(freesurfer_dir, scanid)
try:
run_hcp_convert(path, config, study)
except Exception as e:
logging.error(e)
sys.exit(1)
# run in batch mode
else:
subjects = []
freesurfer_dirs = glob.glob('{}/*'.format(freesurfer_dir))
# find subjects where at least one expected output does not exist
for path in nii_dirs:
subject = os.path.basename(path)
hcp_dir = utils.define_folder(os.path.join(study_base, config.site_config['paths']['hcp']))
if not outputs_exist(subj_dir):
subjects.append(subject)
subjects = list(set(subjects))
# submit a list of calls to ourself, one per subject
commands = []
if debug:
debugopt = '--debug'
else:
debugopt = ''
for subject in subjects:
commands.append(" ".join([__file__, study, '--subject {} '.format(subject), debugopt]))
if commands:
logger.debug('queueing up the following commands:\n'+'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = 'dm_fs2hcp_{}_{}'.format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
#rtn, out = utils.run('echo bash -l {}/{} {} | qbatch -N {} --logdir {} --walltime {} -'.format(bin_dir, script, subid, jobname, logs_dir, walltime))
if rtn:
logger.error("Job submission failed. Output follows.")
logger.error("stdout: {}".format(out))
sys.exit(1)
# Necessary to silence all logging from dm_qc_report during tests.
logging.disable(logging.CRITICAL)
# Turn off testing of integrated dashboard functions for now
datman.dashboard.dash_found = False
qc = importlib.import_module('bin.dm_qc_report')
FIXTURE = "tests/fixture_project_settings"
site_config_path = os.path.join(FIXTURE, 'site_config.yaml')
system = 'local'
study = 'STUDY'
config = cfg.config(filename=site_config_path, system=system, study=study)
class GetConfig(unittest.TestCase):
def test_exits_gracefully_with_bad_study(self):
with pytest.raises(SystemExit):
qc.get_config(study="madeupcode")
@patch('datman.config.config')
def test_exits_gracefully_when_paths_missing_from_config(
self, mock_config):
with pytest.raises(SystemExit):
mock_config.return_value.get_path.side_effect = lambda path: \
{'nii': ''}[path]
qc.get_config("STUDY")
def main():
arguments = docopt(__doc__)
study = arguments['<study>']
subject = arguments['--subject']
debug = arguments['--debug']
logging.info('Starting')
if debug:
logger.setLevel(logging.DEBUG)
# load config for study
try:
config = cfg.config(study=study)
except ValueError:
logger.error('study {} not defined'.format(study))
sys.exit(1)
study_base = config.get_study_base(study)
if 'ea' not in config.study_config['fmri'].keys():
logger.error('ea not defined in fmri in {}'.format(config_file))
sys.exit(1)
for k in ['nii', 'fmri', 'hcp']:
if k not in config.get_key('Paths'):
logger.error("paths:{} not defined in {}".format(k, config_file))
sys.exit(1)
ea_dir = os.path.join(study_base, config.get_path('fmri'), 'ea')
nii_dir = os.path.join(study_base, config.get_path('nii'))
if subject:
if '_PHA_' in subject:
logger.error("{} is a phantom, cannot analyze".format(subject))
sys.exit(1)
analyze_subject(subject, config, study)
else:
# batch mode
subjects = glob.glob('{}/*'.format(nii_dir))
commands = []
if debug:
opts = '--debug'
else:
opts = ''
for path in subjects:
subject = os.path.basename(path)
if check_complete(ea_dir, subject):
logger.debug('{} already analysed'.format(subject))
else:
commands.append(" ".join([__file__, study, '--subject {}'.format(subject), opts]))
if commands:
logger.debug("queueing up the following commands:\n"+'\n'.join(commands))
for i, cmd in enumerate(commands):
jobname = "dm_ea_{}_{}".format(i, time.strftime("%Y%m%d-%H%M%S"))
jobfile = '/tmp/{}'.format(jobname)
logfile = '/tmp/{}.log'.format(jobname)
errfile = '/tmp/{}.err'.format(jobname)
with open(jobfile, 'wb') as fid:
fid.write('#!/bin/bash\n')
fid.write(cmd)
rtn, out = utils.run('qsub -V -q main.q -o {} -e {} -N {} {}'.format(
logfile, errfile, jobname, jobfile))
# qbacth method -- might bring it back, but not needed
#fd, path = tempfile.mkstemp()
#os.write(fd, '\n'.join(commands))
#os.close(fd)
#rtn, out, err = utils.run('qbatch -i --logdir {ld} -N {name} --walltime {wt} {cmds}'.format(ld=logdir, name=jobname, wt=walltime, cmds=path))
if rtn:
logger.error("Job submission failed\nstdout: {}".format(out))
sys.exit(1)
