def dset_info(dset):
'''returns a :class:`DsetInfo` object containing the meta-data from ``dset``'''
if nl.pkg_available('afni'):
return _dset_info_afni(dset)
nl.notify('Error: no packages available to get dset info',
level=nl.level.error)
return None
def unpack_new_archives(pi):
new_files = []
with nl.notify('Scanning PI %s for new archives' % pi):
for root, dirs, files in os.walk(os.path.join(import_location, pi)):
for fname in files + dirs:
full_file = os.path.join(root, fname)
padre_dir = full_file.replace(import_location, "PADRE/Import")
if padre_dir not in import_log:
# Add in a check for the modification date
if any(
[fnmatch.fnmatch(fname, m) for m in c.archive_masks]):
with nl.notify('Found new archive "%s"' % full_file):
subject_guess = os.path.basename(
os.path.dirname(full_file))
nl.notify('guessing the subject number is %s' %
subject_guess)
dsets = import_archive(full_file, subject_guess)
new_files.append({
'dir':
full_file.lstrip(
os.path.join(import_location,
pi)).lstrip('/'),
'subj':
subject_guess,
'fname':
fname,
'dsets':
dsets
})
return new_files
def cdf(dset,p):
info = nl.dset_info(dset)
if info==None:
nl.notify('Error: Could not get info for dset %s'%dset, level=nl.level.error)
return None
command = ['cdf','-p2t',info.subbricks[0]['stat'],str(p)] + info.subbricks[0]['params']
return float(subprocess.check_output(command).split()[2])
def rename(subject_id, new_subject_id):
with commit_wrap():
subj = p.load(subject_id)
if subj:
try:
os.rename(p.subject_dir(subject_id),
p.subject_dir(new_subject_id))
except OSError:
nl.notify('Error: filesystem reported error moving %s to %s' %
(subject_id, new_subject_id),
level=nl.level.error)
else:
subj._subject_id = new_subject_id
subj.save()
if os.path.exists(p.subject_json(subj)):
try:
os.remove(
os.path.join(
p.subject_dir(subj),
os.path.basename(p.subject_json(subject_id))))
except OSError:
pass
try:
del (p.subject._all_subjects[str(subject_id)])
except KeyError:
pass
p.subject._index_one_subject(new_subject_id)
def max_diff(dset1, dset2):
'''calculates maximal voxel-wise difference in datasets (in %)
Useful for checking if datasets have the same data. For example, if the maximum difference is
< 1.0%, they're probably the same dataset'''
for dset in [dset1, dset2]:
if not os.path.exists(dset):
nl.notify('Error: Could not find file: %s' % dset,
level=nl.level.error)
return float('inf')
try:
dset1_d = nib.load(dset1)
dset2_d = nib.load(dset2)
dset1_data = dset1_d.get_data()
dset2_data = dset2_d.get_data()
except IOError:
nl.notify('Error: Could not read files %s and %s' % (dset1, dset2),
level=nl.level.error)
return float('inf')
try:
old_err = np.seterr(divide='ignore', invalid='ignore')
max_val = 100 * np.max(
np.ma.masked_invalid(
np.double(dset1_data - dset2_data) /
((dset1_data + dset2_data) / 2)))
np.seterr(**old_err)
return max_val
except ValueError:
return float('inf')
def pkg_available(pkg_name,required=False):
'''tests if analysis package is available on this machine (e.g., "afni" or "fsl"), and prints an error if ``required``'''
if pkg_name in pkgs:
return True
if required:
nl.notify('Error: could not find required analysis package %s' % pkg_name,level=nl.level.error)
return False
def align_epi_anat(anatomy, epi_dsets, skull_strip_anat=True):
''' aligns epis to anatomy using ``align_epi_anat.py`` script
:epi_dsets: can be either a string or list of strings of the epi child datasets
:skull_strip_anat: if ``True``, ``anatomy`` will be skull-stripped using the default method
The default output suffix is "_al"
'''
if isinstance(epi_dsets, basestring):
epi_dsets = [epi_dsets]
if len(epi_dsets) == 0:
nl.notify('Warning: no epi alignment datasets given for anatomy %s!' %
anatomy,
level=nl.level.warning)
return
if all(os.path.exists(nl.suffix(x, '_al')) for x in epi_dsets):
return
anatomy_use = anatomy
if skull_strip_anat:
nl.skull_strip(anatomy, '_ns')
anatomy_use = nl.suffix(anatomy, '_ns')
inputs = [anatomy_use] + epi_dsets
dset_products = lambda dset: [
nl.suffix(dset, '_al'),
nl.prefix(dset) + '_al_mat.aff12.1D',
nl.prefix(dset) + '_tsh_vr_motion.1D'
]
products = nl.flatten([dset_products(dset) for dset in epi_dsets])
with nl.run_in_tmp(inputs, products):
if nl.is_nifti(anatomy_use):
anatomy_use = nl.afni_copy(anatomy_use)
epi_dsets_use = []
for dset in epi_dsets:
if nl.is_nifti(dset):
epi_dsets_use.append(nl.afni_copy(dset))
else:
epi_dsets_use.append(dset)
cmd = [
"align_epi_anat.py", "-epi2anat", "-anat_has_skull", "no",
"-epi_strip", "3dAutomask", "-anat", anatomy_use, "-epi_base", "5",
"-epi", epi_dsets_use[0]
]
if len(epi_dsets_use) > 1:
cmd += ['-child_epi'] + epi_dsets_use[1:]
out = nl.run(cmd)
for dset in epi_dsets:
if nl.is_nifti(dset):
dset_nifti = nl.nifti_copy(nl.prefix(dset) + '_al+orig')
if dset_nifti and os.path.exists(
dset_nifti) and dset_nifti.endswith(
'.nii') and dset.endswith('.gz'):
nl.run(['gzip', dset_nifti])
def display(name):
try:
user = getpass.getuser()
except:
# Sometimes this seems to fail for no apparent reason
user = "******"
if compress(user) in faces:
nl.notify(decompress(faces[compress(user)][name]))
def pkg_available(pkg_name, required=False):
'''tests if analysis package is available on this machine (e.g., "afni" or "fsl"), and prints an error if ``required``'''
if pkg_name in pkgs:
return True
if required:
nl.notify('Error: could not find required analysis package %s' %
pkg_name,
level=nl.level.error)
return False
def organize_dir(orig_dir):
'''scans through the given directory and organizes DICOMs that look similar into subdirectories
output directory is the ``orig_dir`` with ``-sorted`` appended to the end'''
tags = [
(0x10, 0x20), # Subj ID
(0x8, 0x21), # Date
(0x8, 0x31), # Time
(0x8, 0x103e) # Descr
]
orig_dir = orig_dir.rstrip('/')
files = scan_dir(orig_dir, tags=tags, md5_hash=True)
dups = find_dups(files)
for dup in dups:
nl.notify('Found duplicates of %s...' % dup[0])
for each_dup in dup[1:]:
nl.notify('\tdeleting %s' % each_dup)
try:
os.remove(each_dup)
except IOError:
nl.notify('\t[failed]')
del (files[each_dup])
clustered = cluster_files(files)
output_dir = '%s-sorted' % orig_dir
for key in clustered:
if (0x8, 0x31) in clustered[key]['info']:
clustered[key]['info'][(0x8, 0x31)] = str(
int(float(clustered[key]['info'][(0x8, 0x31)])))
for t in tags:
if t not in clustered[key]['info']:
clustered[key]['info'][t] = '_'
run_name = '-'.join(
[scrub_fname(str(clustered[key]['info'][x]))
for x in tags]) + '-%d_images' % len(clustered[key]['files'])
run_dir = os.path.join(output_dir, run_name)
nl.notify('Moving files into %s' % run_dir)
try:
if not os.path.exists(run_dir):
os.makedirs(run_dir)
except IOError:
nl.notify('Error: failed to create directory %s' % run_dir)
else:
for f in clustered[key]['files']:
try:
dset_fname = os.path.split(f)[1]
if dset_fname[0] == '.':
dset_fname = '_' + dset_fname[1:]
os.rename(f, os.path.join(run_dir, dset_fname))
except (IOError, OSError):
pass
for r, ds, fs in os.walk(output_dir, topdown=False):
for d in ds:
dname = os.path.join(r, d)
if len(os.listdir(dname)) == 0:
os.remove(dname)
def cdf(dset, p):
info = nl.dset_info(dset)
if info == None:
nl.notify('Error: Could not get info for dset %s' % dset,
level=nl.level.error)
return None
command = ['cdf', '-p2t', info.subbricks[0]['stat'],
str(p)] + info.subbricks[0]['params']
return float(subprocess.check_output(command).split()[2])
def available_method(method_name):
'''ruturn the method for earliest package in ``pkg_preferences``, if package is available (based on :meth:`pkg_available`)'''
pkg_prefs_copy = list(pkg_prefs)
if method_name in method_prefs:
pkg_prefs_copy = [method_prefs[method_name]] + pkg_prefs_copy
for pkg in pkg_prefs_copy:
if pkg in pkgs:
if method_name in dir(pkgs[pkg]):
return getattr(pkgs[pkg],method_name)
nl.notify('Error: Could not find implementation of %s on this computer' % (method_name),level=nl.level.error)
def delete_subject(subject_id):
if not os.path.exists(p.trash_dir):
os.makedirs(p.trash_dir)
new_dir = os.path.join(p.trash_dir,'%s-%s' % (subject_id,datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S').format()))
while os.path.exists(new_dir):
new_dir += '_1'
try:
shutil.move(p.subject_dir(subject_id),new_dir)
except OSError,IOError:
nl.notify('Error moving subject directory %s to the trash' % subject_id,level=nl.level.error)
def organize_dir(orig_dir):
'''scans through the given directory and organizes DICOMs that look similar into subdirectories
output directory is the ``orig_dir`` with ``-sorted`` appended to the end'''
tags = [
(0x10,0x20), # Subj ID
(0x8,0x21), # Date
(0x8,0x31), # Time
(0x8,0x103e) # Descr
]
orig_dir = orig_dir.rstrip('/')
files = scan_dir(orig_dir,tags=tags,md5_hash=True)
dups = find_dups(files)
for dup in dups:
nl.notify('Found duplicates of %s...' % dup[0])
for each_dup in dup[1:]:
nl.notify('\tdeleting %s' % each_dup)
try:
os.remove(each_dup)
except IOError:
nl.notify('\t[failed]')
del(files[each_dup])
clustered = cluster_files(files)
output_dir = '%s-sorted' % orig_dir
for key in clustered:
if (0x8,0x31) in clustered[key]['info']:
clustered[key]['info'][(0x8,0x31)] = str(int(float(clustered[key]['info'][(0x8,0x31)])))
for t in tags:
if t not in clustered[key]['info']:
clustered[key]['info'][t] = '_'
run_name = '-'.join([scrub_fname(str(clustered[key]['info'][x])) for x in tags])+'-%d_images' %len(clustered[key]['files'])
run_dir = os.path.join(output_dir,run_name)
nl.notify('Moving files into %s' % run_dir)
try:
if not os.path.exists(run_dir):
os.makedirs(run_dir)
except IOError:
nl.notify('Error: failed to create directory %s' % run_dir)
else:
for f in clustered[key]['files']:
try:
dset_fname = os.path.split(f)[1]
if dset_fname[0]=='.':
dset_fname = '_' + dset_fname[1:]
os.rename(f,os.path.join(run_dir,dset_fname))
except (IOError, OSError):
pass
for r,ds,fs in os.walk(output_dir,topdown=False):
for d in ds:
dname = os.path.join(r,d)
if len(os.listdir(dname))==0:
os.remove(dname)
def available_method(method_name):
'''ruturn the method for earliest package in ``pkg_preferences``, if package is available (based on :meth:`pkg_available`)'''
pkg_prefs_copy = list(pkg_prefs)
if method_name in method_prefs:
pkg_prefs_copy = [method_prefs[method_name]] + pkg_prefs_copy
for pkg in pkg_prefs_copy:
if pkg in pkgs:
if method_name in dir(pkgs[pkg]):
return getattr(pkgs[pkg], method_name)
nl.notify('Error: Could not find implementation of %s on this computer' %
(method_name),
level=nl.level.error)
def error(msg, miss=None):
nl.notify(msg + '\n', level=nl.level.error)
if miss:
try:
import urllib, urllib2
data = {'miss': miss}
data_enc = urllib.urlencode(data)
urllib2.urlopen('http://wolflion.org/cgi-bin/report.py?%s' %
data_enc)
except:
pass
sys.exit()
def dset_copy(dset,to_dir):
'''robust way to copy a dataset (including AFNI briks)'''
if nl.is_afni(dset):
dset_strip = re.sub(r'\.(HEAD|BRIK)?(\.(gz|bz))?','',dset)
for dset_file in [dset_strip + '.HEAD'] + glob.glob(dset_strip + '.BRIK*'):
if os.path.exists(dset_file):
shutil.copy(dset_file,to_dir)
else:
if os.path.exists(dset):
shutil.copy(dset,to_dir)
else:
nl.notify('Warning: couldn\'t find file %s to copy to %s' %(dset,to_dir),level=nl.level.warning)
def time_in(a):
first_number = r'^(\d+(\.\d+)?)'
if isinstance(a,basestring):
m = re.match(first_number,a)
if m:
a = m.group(1)
else:
nl.notify('Warning: cannot intepret a number from the stim_time: "%s"' % a,level=nl.level.warning)
return False
a = float(a)/self.TR
if a>=start and (end==None or a<=end):
return True
return False
def time_in(a):
first_number = r'^(\d+(\.\d+)?)'
if isinstance(a,basestring):
m = re.match(first_number,a)
if m:
a = m.group(1)
else:
nl.notify('Warning: cannot intepret a number from the stim_time: "%s"' % a,level=nl.level.warning)
return False
a = float(a)/self.TR
if a>=start and (end==None or a<=end):
return True
return False
def merge(subject_id_from,subject_id_into):
nl.notify('Trying to merge %s into %s' % (subject_id_from,subject_id_into))
with commit_wrap():
subj_from = p.load(subject_id_from)
subj_to = p.load(subject_id_into)
if subj_from and subj_to:
merge_attr(subj_from.include,subj_to.include)
merge_attr(subj_from.notes,subj_to.notes)
subj_to.meta = dict(subj_from.meta.items() + subj_to.meta.items())
sess_keys = subj_from._sessions.keys()
for sess in sess_keys:
merge_session(subj_from,subj_to,sess)
subj_to.save()
delete_subject(subj_from)
def merge(subject_id_from, subject_id_into):
nl.notify('Trying to merge %s into %s' %
(subject_id_from, subject_id_into))
with commit_wrap():
subj_from = p.load(subject_id_from)
subj_to = p.load(subject_id_into)
if subj_from and subj_to:
merge_attr(subj_from.include, subj_to.include)
merge_attr(subj_from.notes, subj_to.notes)
subj_to.meta = dict(subj_from.meta.items() + subj_to.meta.items())
sess_keys = subj_from._sessions.keys()
for sess in sess_keys:
merge_session(subj_from, subj_to, sess)
subj_to.save()
delete_subject(subj_from)
def nifti_copy(filename,prefix=None,gzip=True):
''' creates a ``.nii`` copy of the given dataset and returns the filename as a string'''
# I know, my argument ``prefix`` clobbers the global method... but it makes my arguments look nice and clean
if prefix==None:
prefix = filename
nifti_filename = globals()['prefix'](prefix) + ".nii"
if gzip:
nifti_filename += '.gz'
if not os.path.exists(nifti_filename):
try:
subprocess.check_call(['3dAFNItoNIFTI','-prefix',nifti_filename,str(filename)])
except subprocess.CalledProcessError:
nl.notify('Error: could not convert "%s" to NIFTI dset!' % filename,level=nl.level.error)
return None
return nifti_filename
def delete_subject(subject_id):
if not os.path.exists(p.trash_dir):
os.makedirs(p.trash_dir)
new_dir = os.path.join(
p.trash_dir, '%s-%s' %
(subject_id,
datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S').format()))
while os.path.exists(new_dir):
new_dir += '_1'
try:
shutil.move(p.subject_dir(subject_id), new_dir)
except OSError, IOError:
nl.notify('Error moving subject directory %s to the trash' %
subject_id,
level=nl.level.error)
def dset_copy(dset, to_dir):
'''robust way to copy a dataset (including AFNI briks)'''
if nl.is_afni(dset):
dset_strip = re.sub(r'\.(HEAD|BRIK)?(\.(gz|bz))?', '', dset)
for dset_file in [dset_strip + '.HEAD'
] + glob.glob(dset_strip + '.BRIK*'):
if os.path.exists(dset_file):
shutil.copy(dset_file, to_dir)
else:
if os.path.exists(dset):
shutil.copy(dset, to_dir)
else:
nl.notify('Warning: couldn\'t find file %s to copy to %s' %
(dset, to_dir),
level=nl.level.warning)
def qwarp_epi(dset, align_subbrick=5, suffix='_qwal', prefix=None):
'''aligns an EPI time-series using 3dQwarp
Very expensive and not efficient at all, but it can produce pretty impressive alignment for EPI time-series with significant
distortions due to motion'''
info = nl.dset_info(dset)
if info == None:
nl.notify('Error reading dataset "%s"' % (dset), level=nl.level.error)
return False
if prefix == None:
prefix = nl.suffix(dset, suffix)
dset_sub = lambda x: '_tmp_qwarp_epi-%s_%d.nii.gz' % (nl.prefix(dset), x)
try:
align_dset = nl.suffix(dset_sub(align_subbrick), '_warp')
nl.calc('%s[%d]' % (dset, align_subbrick),
expr='a',
prefix=align_dset,
datum='float')
for i in xrange(info.reps):
if i != align_subbrick:
nl.calc('%s[%d]' % (dset, i),
expr='a',
prefix=dset_sub(i),
datum='float')
nl.run([
'3dQwarp', '-nowarp', '-workhard', '-superhard',
'-minpatch', '9', '-blur', '0', '-pear', '-nopenalty',
'-base', align_dset, '-source',
dset_sub(i), '-prefix',
nl.suffix(dset_sub(i), '_warp')
],
quiet=True)
cmd = ['3dTcat', '-prefix', prefix]
if info.TR:
cmd += ['-tr', info.TR]
if info.slice_timing:
cmd += ['-tpattern', info.slice_timing]
cmd += [nl.suffix(dset_sub(i), '_warp') for i in xrange(info.reps)]
nl.run(cmd, quiet=True)
except Exception as e:
raise e
finally:
for i in xrange(info.reps):
for suffix in ['', 'warp']:
try:
os.remove(nl.suffix(dset_sub(i), suffix))
except:
pass
def skullstrip_template(dset, template, prefix=None, suffix=None, dilate=0):
'''Takes the raw anatomy ``dset``, aligns it to a template brain, and applies a templated skullstrip. Should produce fairly reliable skullstrips as long
as there is a decent amount of normal brain and the overall shape of the brain is normal-ish'''
if suffix == None:
suffix = '_sstemplate'
if prefix == None:
prefix = nl.suffix(dset, suffix)
if not os.path.exists(prefix):
with nl.notify('Running template-based skull-strip on %s' % dset):
dset = os.path.abspath(dset)
template = os.path.abspath(template)
tmp_dir = tempfile.mkdtemp()
cwd = os.getcwd()
with nl.run_in(tmp_dir):
nl.affine_align(template,
dset,
skull_strip=None,
cost='mi',
opts=['-nmatch', '100%'])
nl.run([
'3dQwarp', '-minpatch', '20', '-penfac', '10', '-noweight',
'-source',
nl.suffix(template, '_aff'), '-base', dset, '-prefix',
nl.suffix(template, '_qwarp')
],
products=nl.suffix(template, '_qwarp'))
info = nl.dset_info(nl.suffix(template, '_qwarp'))
max_value = info.subbricks[0]['max']
nl.calc([dset, nl.suffix(template, '_qwarp')],
'a*step(b-%f*0.05)' % max_value, prefix)
shutil.move(prefix, cwd)
shutil.rmtree(tmp_dir)
def convert_coord(coord_from,matrix_file,base_to_aligned=True):
'''Takes an XYZ array (in DICOM coordinates) and uses the matrix file produced by 3dAllineate to transform it. By default, the 3dAllineate
matrix transforms from base to aligned space; to get the inverse transform set ``base_to_aligned`` to ``False``'''
with open(matrix_file) as f:
try:
values = [float(y) for y in ' '.join([x for x in f.readlines() if x.strip()[0]!='#']).strip().split()]
except:
nl.notify('Error reading values from matrix file %s' % matrix_file, level=nl.level.error)
return False
if len(values)!=12:
nl.notify('Error: found %d values in matrix file %s (expecting 12)' % (len(values),matrix_file), level=nl.level.error)
return False
matrix = np.vstack((np.array(values).reshape((3,-1)),[0,0,0,1]))
if not base_to_aligned:
matrix = np.linalg.inv(matrix)
return np.dot(matrix,list(coord_from) + [1])[:3]
def align_epi_anat(anatomy,epi_dsets,skull_strip_anat=True):
''' aligns epis to anatomy using ``align_epi_anat.py`` script
:epi_dsets: can be either a string or list of strings of the epi child datasets
:skull_strip_anat: if ``True``, ``anatomy`` will be skull-stripped using the default method
The default output suffix is "_al"
'''
if isinstance(epi_dsets,basestring):
epi_dsets = [epi_dsets]
if len(epi_dsets)==0:
nl.notify('Warning: no epi alignment datasets given for anatomy %s!' % anatomy,level=nl.level.warning)
return
if all(os.path.exists(nl.suffix(x,'_al')) for x in epi_dsets):
return
anatomy_use = anatomy
if skull_strip_anat:
nl.skull_strip(anatomy,'_ns')
anatomy_use = nl.suffix(anatomy,'_ns')
inputs = [anatomy_use] + epi_dsets
dset_products = lambda dset: [nl.suffix(dset,'_al'), nl.prefix(dset)+'_al_mat.aff12.1D', nl.prefix(dset)+'_tsh_vr_motion.1D']
products = nl.flatten([dset_products(dset) for dset in epi_dsets])
with nl.run_in_tmp(inputs,products):
if nl.is_nifti(anatomy_use):
anatomy_use = nl.afni_copy(anatomy_use)
epi_dsets_use = []
for dset in epi_dsets:
if nl.is_nifti(dset):
epi_dsets_use.append(nl.afni_copy(dset))
else:
epi_dsets_use.append(dset)
cmd = ["align_epi_anat.py", "-epi2anat", "-anat_has_skull", "no", "-epi_strip", "3dAutomask","-anat", anatomy_use, "-epi_base", "5", "-epi", epi_dsets_use[0]]
if len(epi_dsets_use)>1:
cmd += ['-child_epi'] + epi_dsets_use[1:]
out = nl.run(cmd)
for dset in epi_dsets:
if nl.is_nifti(dset):
dset_nifti = nl.nifti_copy(nl.prefix(dset)+'_al+orig')
if dset_nifti and os.path.exists(dset_nifti) and dset_nifti.endswith('.nii') and dset.endswith('.gz'):
nl.run(['gzip',dset_nifti])
def new_object(args):
args = [
x for x in args
if 'concept' not in dir(x[0]) or x[0].concept.name != 'new'
]
concept_names = [x[0].concept.name for x in args if 'concept' in dir(x[0])]
if 'session_do' in concept_names:
other_args = [x for x in args if isinstance(x, basestring)]
if len(other_args) > 1:
nl.notify(
'Error: You gave me multiple names (%s), don\'t know which you want to be the session name!'
% ','.join(other_args),
level=nl.level.error)
return False
new_session = other_args[0]
if 'subject' not in concept_names:
nl.notify(
'Error: I don\'t know what subject to create session %s in!' %
new_session,
level=nl.level.error)
return False
subject_id = [
x[0].item for x in args
if 'concept' in dir(x[0]) and x[0].concept.name == 'subject'
][0]
with nl.notify('Creating session %s in subject %s' %
(new_session, subject_id)):
subj = p.load(subject_id)
if not subj:
nl.notify('Error: couldn\'t load data for subject "%s"' %
subject_id,
level=nl.level.error)
return False
p.maint.new_session(subj, new_session)
return True
# Otherwise assume you want to make a subject
other_args = [x for x in args if isinstance(x, basestring)]
if len(other_args) > 1:
nl.notify(
'Error: You gave me multiple names (%s), don\'t know which you want to be the subject name!'
% ','.join(other_args),
level=nl.level.error)
return False
new_subject = other_args[0]
with nl.notify('Creating new subject %s' % (new_subject)):
p.maint.create_subject(new_subject)
return True
def dsets_identical(dset1, dset2):
'''Tests if given datasets are identical'''
max_tolerance = 1.0
with nl.notify('Comparing %s with %s' % (dset1, dset2)):
# info = [nl.afni.dset_info(dset) for dset in [dset1,dset2]]
# for param in ['reps','voxel_size','voxel_dims']:
# if getattr(info[0],param) != getattr(info[1],param):
# nl.notify('Datasets differ in at least %s (%s vs. %s)' % (param,getattr(info[0],param),getattr(info[1],param)),level=nl.level.warning)
# return False
max_diff = nl.max_diff(dset1, dset2)
if max_diff > max_tolerance:
nl.notify(
'Datasets have a maximal differenence >%.1f (max_diff = %.1f)'
% (max_tolerance, max_diff),
level=nl.level.warning)
return False
if max_diff == 0:
nl.notify('Datasets appear to be identical')
return True
nl.notify('Datasets are minimally different (max_diff = %.1f)' %
max_diff)
return True
def reconstruct_files(input_dir):
'''sorts ``input_dir`` and tries to reconstruct the subdirectories found'''
input_dir = input_dir.rstrip('/')
with nl.notify('Attempting to organize/reconstruct directory'):
# Some datasets start with a ".", which confuses many programs
for r,ds,fs in os.walk(input_dir):
for f in fs:
if f[0]=='.':
shutil.move(os.path.join(r,f),os.path.join(r,'i'+f))
nl.dicom.organize_dir(input_dir)
output_dir = '%s-sorted' % input_dir
if os.path.exists(output_dir):
with nl.run_in(output_dir):
for dset_dir in os.listdir('.'):
with nl.notify('creating dataset from %s' % dset_dir):
nl.dicom.create_dset(dset_dir)
else:
nl.notify('Warning: failed to auto-organize directory %s' % input_dir,level=nl.level.warning)
def create_censor_file(input_dset,out_prefix=None,fraction=0.1,clip_to=0.1,max_exclude=0.3,motion_file=None,motion_exclude=1.0):
'''create a binary censor file using 3dToutcount
:input_dset: the input dataset
:prefix: output 1D file (default: ``prefix(input_dset)`` + ``.1D``)
:fraction: censor a timepoint if proportional of outliers in this
time point is greater than given value
:clip_to: keep the number of time points censored under this proportion
of total reps. If more time points would be censored,
it will only pick the top ``clip_to*reps`` points
:max_exclude: if more time points than the given proportion of reps are excluded for the
entire run, throw an exception -- something is probably wrong
:motion_file: optional filename of a "motion" file with multiple columns and rows corresponding to reps.
It doesn't really matter what the values are, as long as they are appropriate relative to ``motion_exclude``
:motion_exclude: Will exclude any reps that have a value greater than this in any column of ``motion_file``
'''
(outcount,perc_outliers) = nl.qc.outcount(input_dset,fraction)
info = nl.dset_info(input_dset)
binarize = lambda o,f: [oo<f for oo in o]
perc_outliers = lambda o: 1.-(sum(o)/float(info.reps))
if motion_file:
with open(motion_file,'Ur') as f:
motion = [max([float(y) for y in x.strip().split()]) for x in f.read().split('\n') if len(x.strip())>0 and x.strip()[0]!='#']
motion_1D = [x for x in binarize(motion,motion_exclude)]
if perc_outliers(motion_1D) > max_exclude:
nl.notify('Error: Too many points excluded because of motion (%.2f) in dset %s' % (perc_outliers(motion_1D),input_dset),level=nl.level.error)
return False
outcount = [outcount[i] if motion_1D[i] else 1. for i in range(len(outcount))]
binary_outcount = binarize(outcount,fraction)
if max_exclude and perc_outliers(binary_outcount) > max_exclude:
nl.notify('Error: Found %.1f%% outliers in dset %s' % (100*perc_outliers(outcount),input_dset),level=nl.level.error)
return False
if clip_to:
while perc_outliers(binary_outcount) > clip_to:
best_outlier = min([(outcount[i],i) for i in range(len(outcount)) if not binary_outcount[i]])
binary_outcount[best_outlier[1]] = True
if not out_prefix:
out_prefix = nl.prefix(input_dset) + '.1D'
with open(out_prefix,'w') as f:
f.write('\n'.join([str(int(x)) for x in binary_outcount]))
return True
def nifti_copy(filename, prefix=None, gzip=True):
''' creates a ``.nii`` copy of the given dataset and returns the filename as a string'''
# I know, my argument ``prefix`` clobbers the global method... but it makes my arguments look nice and clean
if prefix == None:
prefix = filename
nifti_filename = globals()['prefix'](prefix) + ".nii"
if gzip:
nifti_filename += '.gz'
if not os.path.exists(nifti_filename):
try:
subprocess.check_call(
['3dAFNItoNIFTI', '-prefix', nifti_filename,
str(filename)])
except subprocess.CalledProcessError:
nl.notify('Error: could not convert "%s" to NIFTI dset!' %
filename,
level=nl.level.error)
return None
return nifti_filename
def subbrick(dset,
label,
coef=False,
tstat=False,
fstat=False,
rstat=False,
number_only=False):
''' returns a string referencing the given subbrick within a dset
This method reads the header of the dataset ``dset``, finds the subbrick whose
label matches ``label`` and returns a string of type ``dataset[X]``, which can
be used by most AFNI programs to refer to a subbrick within a file
The options coef, tstat, fstat, and rstat will add the suffix that is
appended to the label by 3dDeconvolve
:coef: "#0_Coef"
:tstat: "#0_Tstat"
:fstat: "_Fstat"
:rstat: "_R^2"
if ``number_only`` is set to ``True``, will only return the subbrick number instead of a string
'''
if coef:
label += "#0_Coef"
elif tstat:
label += "#0_Tstat"
elif fstat:
label += "_Fstat"
elif rstat:
label += "_R^2"
info = nl.dset_info(dset)
if info == None:
nl.notify('Error: Couldn\'t get info from dset "%s"' % dset,
level=nl.level.error)
return None
i = info.subbrick_labeled(label)
if number_only:
return i
return '%s[%d]' % (dset, i)
def partial(self,start=0,end=None,run=0):
'''chops the stimulus by only including time points ``start`` through ``end`` (in reps, inclusive; ``None``=until the end)
if using stim_times-style simulus, will change the ``run``'th run. If a column, will just chop the column'''
self.read_file()
decon_stim = copy.copy(self)
if start<0:
start = 0
if self.type()=="column":
decon_stim.column_file = None
if end>=len(decon_stim.column):
end = None
if end==None:
decon_stim.column = decon_stim.column[start:]
else:
decon_stim.column = decon_stim.column[start:end+1]
if len(decon_stim.column)==0:
return None
if self.type()=="times":
if self.TR==None:
nl.notify('Error: cannot get partial segment of a stim_times stimulus without a TR',level=nl.level.error)
return None
def time_in(a):
first_number = r'^(\d+(\.\d+)?)'
if isinstance(a,basestring):
m = re.match(first_number,a)
if m:
a = m.group(1)
else:
nl.notify('Warning: cannot intepret a number from the stim_time: "%s"' % a,level=nl.level.warning)
return False
a = float(a)/self.TR
if a>=start and (end==None or a<=end):
return True
return False
decon_stim.times_file = None
if len(decon_stim.times)==0 or '__iter__' not in dir(decon_stim.times[0]):
decon_stim.times = [decon_stim.times]
decon_stim.times[run] = [x for x in decon_stim.times[run] if time_in(x)]
if len(nl.flatten(decon_stim.times))==0:
return None
return decon_stim
def subbrick(dset,label,coef=False,tstat=False,fstat=False,rstat=False,number_only=False):
''' returns a string referencing the given subbrick within a dset
This method reads the header of the dataset ``dset``, finds the subbrick whose
label matches ``label`` and returns a string of type ``dataset[X]``, which can
be used by most AFNI programs to refer to a subbrick within a file
The options coef, tstat, fstat, and rstat will add the suffix that is
appended to the label by 3dDeconvolve
:coef: "#0_Coef"
:tstat: "#0_Tstat"
:fstat: "_Fstat"
:rstat: "_R^2"
If ``coef`` or ``tstat`` are set to a number, it will use that parameter number
(instead of 0), for models that use multiple parameters (e.g., "TENT").
if ``number_only`` is set to ``True``, will only return the subbrick number instead of a string
'''
if coef is not False:
if coef is True:
coef = 0
label += "#%d_Coef" % coef
elif tstat != False:
if tstat==True:
tstat = 0
label += "#%d_Tstat" % tstat
elif fstat:
label += "_Fstat"
elif rstat:
label += "_R^2"
info = nl.dset_info(dset)
if info==None:
nl.notify('Error: Couldn\'t get info from dset "%s"'%dset,level=nl.level.error)
return None
i = info.subbrick_labeled(label)
if number_only:
return i
return '%s[%d]' % (dset,i)
def qwarp_epi(dset,align_subbrick=5,suffix='_qwal',prefix=None):
'''aligns an EPI time-series using 3dQwarp
Very expensive and not efficient at all, but it can produce pretty impressive alignment for EPI time-series with significant
distortions due to motion'''
info = nl.dset_info(dset)
if info==None:
nl.notify('Error reading dataset "%s"' % (dset),level=nl.level.error)
return False
if prefix==None:
prefix = nl.suffix(dset,suffix)
dset_sub = lambda x: '_tmp_qwarp_epi-%s_%d.nii.gz' % (nl.prefix(dset),x)
try:
align_dset = nl.suffix(dset_sub(align_subbrick),'_warp')
nl.calc('%s[%d]' % (dset,align_subbrick),expr='a',prefix=align_dset,datum='float')
for i in xrange(info.reps):
if i != align_subbrick:
nl.calc('%s[%d]' % (dset,i),expr='a',prefix=dset_sub(i),datum='float')
nl.run([
'3dQwarp', '-nowarp',
'-workhard', '-superhard', '-minpatch', '9', '-blur', '0',
'-pear', '-nopenalty',
'-base', align_dset,
'-source', dset_sub(i),
'-prefix', nl.suffix(dset_sub(i),'_warp')
],quiet=True)
cmd = ['3dTcat','-prefix',prefix]
if info.TR:
cmd += ['-tr',info.TR]
if info.slice_timing:
cmd += ['-tpattern',info.slice_timing]
cmd += [nl.suffix(dset_sub(i),'_warp') for i in xrange(info.reps)]
nl.run(cmd,quiet=True)
except Exception as e:
raise e
finally:
for i in xrange(info.reps):
for suffix in ['','warp']:
try:
os.remove(nl.suffix(dset_sub(i),suffix))
except:
pass
def partial(self,start=0,end=None,run=0):
'''chops the stimulus by only including time points ``start`` through ``end`` (in reps, inclusive; ``None``=until the end)
if using stim_times-style simulus, will change the ``run``'th run. If a column, will just chop the column'''
self.read_file()
decon_stim = copy.copy(self)
if start<0:
start = 0
if self.type()=="column":
decon_stim.column_file = None
if end>=len(decon_stim.column):
end = None
if end==None:
decon_stim.column = decon_stim.column[start:]
else:
decon_stim.column = decon_stim.column[start:end+1]
if len(decon_stim.column)==0:
return None
if self.type()=="times":
if self.TR==None:
nl.notify('Error: cannot get partial segment of a stim_times stimulus without a TR',level=nl.level.error)
return None
def time_in(a):
first_number = r'^(\d+(\.\d+)?)'
if isinstance(a,basestring):
m = re.match(first_number,a)
if m:
a = m.group(1)
else:
nl.notify('Warning: cannot intepret a number from the stim_time: "%s"' % a,level=nl.level.warning)
return False
a = float(a)/self.TR
if a>=start and (end==None or a<=end):
return True
return False
decon_stim.times_file = None
if len(decon_stim.times)==0 or '__iter__' not in dir(decon_stim.times[0]):
decon_stim.times = [decon_stim.times]
decon_stim.times[run] = [x for x in decon_stim.times[run] if time_in(x)]
if len(nl.flatten(decon_stim.times))==0:
return None
return decon_stim
def reconstruct_files(input_dir):
'''sorts ``input_dir`` and tries to reconstruct the subdirectories found'''
input_dir = input_dir.rstrip('/')
with nl.notify('Attempting to organize/reconstruct directory'):
# Some datasets start with a ".", which confuses many programs
for r, ds, fs in os.walk(input_dir):
for f in fs:
if f[0] == '.':
shutil.move(os.path.join(r, f), os.path.join(r, 'i' + f))
nl.dicom.organize_dir(input_dir)
output_dir = '%s-sorted' % input_dir
if os.path.exists(output_dir):
with nl.run_in(output_dir):
for dset_dir in os.listdir('.'):
with nl.notify('creating dataset from %s' % dset_dir):
nl.dicom.create_dset(dset_dir)
else:
nl.notify('Warning: failed to auto-organize directory %s' %
input_dir,
level=nl.level.warning)
def blank_stim(self,type=None,fill=0):
'''Makes a blank version of stim. If a type is not given, returned as same type as current stim.
If a column stim, will fill in blanks with ``fill``'''
blank = copy.copy(self)
blank.name = 'Blank'
if type==None:
type = self.type()
if type=="column":
num_reps = self.reps
if num_reps==None:
if self.type()=="column":
self.read_file()
num_reps = len(self.column)
else:
nl.notify('Error: requested to return a blank column, but I can\'t figure out how many reps to make it!',level=nl.level.error)
blank.column = [fill]*num_reps
return blank
if type=="times":
blank.times = []
return blank
def blank_stim(self,type=None,fill=0):
'''Makes a blank version of stim. If a type is not given, returned as same type as current stim.
If a column stim, will fill in blanks with ``fill``'''
blank = copy.copy(self)
blank.name = 'Blank'
if type==None:
type = self.type()
if type=="column":
num_reps = self.reps
if num_reps==None:
if self.type()=="column":
self.read_file()
num_reps = len(self.column)
else:
nl.notify('Error: requested to return a blank column, but I can\'t figure out how many reps to make it!',level=nl.level.error)
blank.column = [fill]*num_reps
return blank
if type=="times":
blank.times = []
return blank
def sessions_identical(subj1,sess1,subj2,sess2):
'''Tests the given sessions to make sure the datasets are the same'''
dsets1 = [os.path.basename(str(x)) for x in subj1.dsets(session=sess1)]
dsets2 = [os.path.basename(str(x)) for x in subj2.dsets(session=sess2)]
dsets = list(set(dsets1+dsets2))
return_val = True
with nl.notify('Comparing sessions %s.%s and %s.%s:' % (subj1,sess1,subj2,sess2)):
for dset in dsets:
if not dsets_identical(os.path.join(p.sessions_dir(subj1),sess1,dset),os.path.join(p.sessions_dir(subj2),sess2,dset)):
return_val = False
continue
return return_val
def rename(subject_id,new_subject_id):
with commit_wrap():
subj = p.load(subject_id)
if subj:
try:
os.rename(p.subject_dir(subject_id),p.subject_dir(new_subject_id))
except OSError:
nl.notify('Error: filesystem reported error moving %s to %s' % (subject_id,new_subject_id),level=nl.level.error)
else:
subj._subject_id = new_subject_id
subj.save()
if os.path.exists(p.subject_json(subj)):
try:
os.remove(os.path.join(p.subject_dir(subj),os.path.basename(p.subject_json(subject_id))))
except OSError:
pass
try:
del(p.subject._all_subjects[str(subject_id)])
except KeyError:
pass
p.subject._index_one_subject(new_subject_id)
def filter_subjs(subjects=None, string=None, matches=None, require_match=True):
'''takes ``list`` of subjects and filters based on the :class:`ConceptMatch`s given in ``matches``. If ``matches``
is not given, will parse the string ``string`` instead. If ``require_match``, will return ``None`` if it fails to find
any constraints, otherwise it returns ``subjects``'''
match_concepts = ['subject', 'session', 'label', 'experiment', 'tag']
if matches == None:
if string == None:
if subjects == None:
return p.subjects()
else:
return subjects
matches = bottle.parse_string(string)
if subjects == None:
if 'all' in [x[0].concept.name for x in matches]:
nl.notify('Including incomplete sessions')
subjects = p.subjects(only_included=False)
else:
subjects = p.subjects()
running_exclusions = {}
if not any([[x in sub_matches for x in match_concepts]
for sub_matches in matches]):
if require_match:
nl.notify('Warning: Could not find any constraints to use',
level=nl.level.warning)
return None
else:
nl.notify('Using no constraints')
return subjects
with nl.notify('Using constraints:'):
subjects_total = set(subjects)
for match_options in matches:
if not isinstance(match_options, list):
continue
match_options = [x for x in match_options if x in match_concepts]
if len(match_options) == 0:
continue
subjects_options = set()
for match in match_options:
nl.notify('%s = %s' %
(match.concept.name, repr(match.examples)))
subjects_options = logic.or_sets(
subjects_options, set(filter_by_match(match, subjects)))
subjects_total = logic.and_sets(subjects_options, subjects_total)
return subjects_total
def link_dsets(args):
with nl.notify('Trying to link the following datasets...'):
subjects = filter_subjs(p.subjects(), args)
dsets_to_link = []
for subj in subjects:
with nl.notify(str(subj)):
for dset in dsets_with(subj, args):
nl.notify(dset.__str__(False))
dsets_to_link.append(dset)
nl.notify('Does that all look good? (y/n)')
i = raw_input()
if i.lower() == 'y':
nl.notify('Ok, linking them')
for dset in dsets_to_link:
try:
os.symlink(str(dset), os.path.basename(str(dset)))
except OSError:
pass
else:
nl.notify('Nope, that didn\'t look like a "y"...',
level=nl.level.warning)
def bottle_help():
nl.notify('''This program uses freeform text fuzzy matching. It will try to correct for misspellings and
synonyms as it can.
General syntax:
[action] [objects]
Currently supported actions are:
list print something to the screen (add quiet to do it less loud)
add add on something to an existing subject
new create a new subject
link create symbolic links to datasets in current directory
Examples:
list subjects
list subjects ExperimentName
get dsets SubjectName
get subjects LabelName Tag1
Just try stuff...
''')
def concat_stim(self,decon_stim):
'''concatenate this to another :class:`DeconStim` of the same "type"'''
if self.type()!=decon_stim.type():
nl.notify('Error: Trying to concatenate stimuli of different types! %s (%s) with %s (%s)' % (self.name,self.type(),decon_stim.name,decon_stim.type()),level=nl.level.error)
return None
concat_stim = copy.copy(self)
if self.name=='Blank':
concat_stim = copy.copy(decon_stim)
self.read_file()
if self.type()=="column":
# if an explicit # of reps is given, concat to that
reps = [x.reps if x.reps else len(x.column) for x in [self,decon_stim]]
concat_stim.column = self.column[:reps[0]] + decon_stim.column[:reps[1]]
return concat_stim
if self.type()=="times":
if len(self.times)==0 or '__iter__' not in dir(self.times[0]):
self.times = [self.times]
if len(decon_stim.times)==0 or '__iter__' not in dir(decon_stim.times[0]):
decon_stim.times = [decon_stim.times]
concat_stim.times = self.times + decon_stim.times
return concat_stim
return None
def convert_coord(coord_from, matrix_file, base_to_aligned=True):
'''Takes an XYZ array (in DICOM coordinates) and uses the matrix file produced by 3dAllineate to transform it. By default, the 3dAllineate
matrix transforms from base to aligned space; to get the inverse transform set ``base_to_aligned`` to ``False``'''
with open(matrix_file) as f:
try:
values = [
float(y) for y in ' '.join(
[x for x in f.readlines()
if x.strip()[0] != '#']).strip().split()
]
except:
nl.notify('Error reading values from matrix file %s' % matrix_file,
level=nl.level.error)
return False
if len(values) != 12:
nl.notify('Error: found %d values in matrix file %s (expecting 12)' %
(len(values), matrix_file),
level=nl.level.error)
return False
matrix = np.vstack((np.array(values).reshape((3, -1)), [0, 0, 0, 1]))
if not base_to_aligned:
matrix = np.linalg.inv(matrix)
return np.dot(matrix, list(coord_from) + [1])[:3]
def concat_stim(self,decon_stim):
'''concatenate this to another :class:`DeconStim` of the same "type"'''
if self.type()!=decon_stim.type():
nl.notify('Error: Trying to concatenate stimuli of different types! %s (%s) with %s (%s)' % (self.name,self.type(),decon_stim.name,decon_stim.type()),level=nl.level.error)
return None
concat_stim = copy.copy(self)
if self.name=='Blank':
concat_stim = copy.copy(decon_stim)
self.read_file()
if self.type()=="column":
# if an explicit # of reps is given, concat to that
reps = [x.reps if x.reps else len(x.column) for x in [self,decon_stim]]
concat_stim.column = self.column[:reps[0]] + decon_stim.column[:reps[1]]
return concat_stim
if self.type()=="times":
if len(self.times)==0 or '__iter__' not in dir(self.times[0]):
self.times = [self.times]
if len(decon_stim.times)==0 or '__iter__' not in dir(decon_stim.times[0]):
decon_stim.times = [decon_stim.times]
concat_stim.times = self.times + decon_stim.times
return concat_stim
return None
def bottle_help():
nl.notify(
'''This program uses freeform text fuzzy matching. It will try to correct for misspellings and
synonyms as it can.
General syntax:
[action] [objects]
Currently supported actions are:
list print something to the screen (add quiet to do it less loud)
add add on something to an existing subject
new create a new subject
link create symbolic links to datasets in current directory
Examples:
list subjects
list subjects ExperimentName
get dsets SubjectName
get subjects LabelName Tag1
Just try stuff...
''')
def stack_decon_stims(stim_list):
'''take a ``list`` (in order of runs) of ``dict``s of stim_name:DeconStim and stack them together. returns
a single ``dict`` of stim_name:decon_stim
As in, takes:
[
# Run 1
{ "stim1": decon_stim1a, "stim2": decon_stim2a },
# Run 2
{ "stim1": decon_stim1b, "stim2": decon_stim2b, "stim3": decon_stim3 }
]
And makes:
{ "stim1": decon_stim1, "stim2": decon_stim2, "stim3": decon_stim3 }
If a stimulus is not present in a run, it will fill that run with an empty stimulus
'''
stim_names = list(set(nl.flatten([stims.keys() for stims in stim_list])))
stim_dict = {}
for stim_name in stim_names:
types = list(set([stims[stim_name].type() for stims in stim_list if stim_name in stims]))
if len(types)>1:
nl.notify('Error: Trying to stack stimuli of different types! (%s)' % stim_name,level=nl.level.error)
return None
type = types[0]
stim_stack = []
for i in xrange(len(stim_list)):
if stim_name in stim_list[i]:
stim_stack.append(stim_list[i][stim_name])
else:
stim_stack.append(stim_list[i].values()[0].blank_stim(type=type))
stim_dict[stim_name] = copy.copy(stim_stack[0])
for stim in stim_stack[1:]:
stim_dict[stim_name] = stim_dict[stim_name].concat_stim(stim)
return stim_dict.values()
def stack_decon_stims(stim_list):
'''take a ``list`` (in order of runs) of ``dict``s of stim_name:DeconStim and stack them together. returns
a single ``dict`` of stim_name:decon_stim
As in, takes:
[
# Run 1
{ "stim1": decon_stim1a, "stim2": decon_stim2a },
# Run 2
{ "stim1": decon_stim1b, "stim2": decon_stim2b, "stim3": decon_stim3 }
]
And makes:
{ "stim1": decon_stim1, "stim2": decon_stim2, "stim3": decon_stim3 }
If a stimulus is not present in a run, it will fill that run with an empty stimulus
'''
stim_names = list(set(nl.flatten([stims.keys() for stims in stim_list])))
stim_dict = {}
for stim_name in stim_names:
types = list(set([stims[stim_name].type() for stims in stim_list if stim_name in stims]))
if len(types)>1:
nl.notify('Error: Trying to stack stimuli of different types! (%s)' % stim_name,level=nl.level.error)
return None
type = types[0]
stim_stack = []
for i in xrange(len(stim_list)):
if stim_name in stim_list[i]:
stim_stack.append(stim_list[i][stim_name])
else:
stim_stack.append(stim_list[i].values()[0].blank_stim(type=type))
stim_dict[stim_name] = copy.copy(stim_stack[0])
for stim in stim_stack[1:]:
stim_dict[stim_name] = stim_dict[stim_name].concat_stim(stim)
return stim_dict.values()
def filter_subjs(subjects=None,string=None,matches=None,require_match=True):
'''takes ``list`` of subjects and filters based on the :class:`ConceptMatch`s given in ``matches``. If ``matches``
is not given, will parse the string ``string`` instead. If ``require_match``, will return ``None`` if it fails to find
any constraints, otherwise it returns ``subjects``'''
match_concepts = ['subject','session','label','experiment','tag']
if matches==None:
if string==None:
if subjects==None:
return p.subjects()
else:
return subjects
matches = bottle.parse_string(string)
if subjects==None:
if 'all' in [x[0].concept.name for x in matches]:
nl.notify('Including incomplete sessions')
subjects = p.subjects(only_included=False)
else:
subjects = p.subjects()
running_exclusions = {}
if not any([[x in sub_matches for x in match_concepts] for sub_matches in matches]):
if require_match:
nl.notify('Warning: Could not find any constraints to use',level=nl.level.warning)
return None
else:
nl.notify('Using no constraints')
return subjects
with nl.notify('Using constraints:'):
subjects_total = set(subjects)
for match_options in matches:
if not isinstance(match_options,list):
continue
match_options = [x for x in match_options if x in match_concepts]
if len(match_options)==0:
continue
subjects_options = set()
for match in match_options:
nl.notify('%s = %s' % (match.concept.name,repr(match.examples)))
subjects_options = logic.or_sets(subjects_options,set(filter_by_match(match,subjects)))
subjects_total = logic.and_sets(subjects_options,subjects_total)
return subjects_total
def max_diff(dset1,dset2):
'''calculates maximal voxel-wise difference in datasets (in %)
Useful for checking if datasets have the same data. For example, if the maximum difference is
< 1.0%, they're probably the same dataset'''
for dset in [dset1,dset2]:
if not os.path.exists(dset):
nl.notify('Error: Could not find file: %s' % dset,level=nl.level.error)
return float('inf')
try:
dset1_d = nib.load(dset1)
dset2_d = nib.load(dset2)
dset1_data = dset1_d.get_data()
dset2_data = dset2_d.get_data()
except IOError:
nl.notify('Error: Could not read files %s and %s' % (dset1,dset2),level=nl.level.error)
return float('inf')
try:
old_err = np.seterr(divide='ignore',invalid='ignore')
max_val = 100*np.max(np.ma.masked_invalid(np.double(dset1_data - dset2_data) / ((dset1_data+dset2_data)/2)))
np.seterr(**old_err)
return max_val
except ValueError:
return float('inf')
def skull_strip(dset, suffix='_ns', prefix=None, unifize=True):
''' use bet to strip skull from given anatomy '''
# should add options to use betsurf and T1/T2 in the future
# Since BET fails on weirdly distributed datasets, I added 3dUnifize in... I realize this makes this dependent on AFNI. Sorry, :)
if prefix == None:
prefix = nl.suffix(dset, suffix)
unifize_dset = nl.suffix(dset, '_u')
cmd = bet2 if bet2 else 'bet2'
if unifize:
info = nl.dset_info(dset)
if info == None:
nl.notify('Error: could not read info for dset %s' % dset,
level=nl.level.error)
return False
cmd = os.path.join(fsl_dir, cmd) if fsl_dir else cmd
cutoff_value = nl.max(dset) * 0.05
nl.run([
'3dUnifize', '-prefix', unifize_dset,
nl.calc(dset, 'step(a-%f)*a' % cutoff_value)
],
products=unifize_dset)
else:
unifize_dset = dset
nl.run([cmd, unifize_dset, prefix, '-w', 0.5], products=prefix)
def skullstrip_template(dset,template,prefix=None,suffix=None,dilate=0):
'''Takes the raw anatomy ``dset``, aligns it to a template brain, and applies a templated skullstrip. Should produce fairly reliable skullstrips as long
as there is a decent amount of normal brain and the overall shape of the brain is normal-ish'''
if suffix==None:
suffix = '_sstemplate'
if prefix==None:
prefix = nl.suffix(dset,suffix)
if not os.path.exists(prefix):
with nl.notify('Running template-based skull-strip on %s' % dset):
dset = os.path.abspath(dset)
template = os.path.abspath(template)
tmp_dir = tempfile.mkdtemp()
cwd = os.getcwd()
with nl.run_in(tmp_dir):
nl.affine_align(template,dset,skull_strip=None,cost='mi',opts=['-nmatch','100%'])
nl.run(['3dQwarp','-minpatch','20','-penfac','10','-noweight','-source',nl.suffix(template,'_aff'),'-base',dset,'-prefix',nl.suffix(template,'_qwarp')],products=nl.suffix(template,'_qwarp'))
info = nl.dset_info(nl.suffix(template,'_qwarp'))
max_value = info.subbricks[0]['max']
nl.calc([dset,nl.suffix(template,'_qwarp')],'a*step(b-%f*0.05)'%max_value,prefix)
shutil.move(prefix,cwd)
shutil.rmtree(tmp_dir)
def dsets_identical(dset1,dset2):
'''Tests if given datasets are identical'''
max_tolerance = 1.0
with nl.notify('Comparing %s with %s' % (dset1,dset2)):
# info = [nl.afni.dset_info(dset) for dset in [dset1,dset2]]
# for param in ['reps','voxel_size','voxel_dims']:
# if getattr(info[0],param) != getattr(info[1],param):
# nl.notify('Datasets differ in at least %s (%s vs. %s)' % (param,getattr(info[0],param),getattr(info[1],param)),level=nl.level.warning)
# return False
max_diff = nl.max_diff(dset1,dset2)
if max_diff > max_tolerance:
nl.notify('Datasets have a maximal differenence >%.1f (max_diff = %.1f)' % (max_tolerance, max_diff),level=nl.level.warning)
return False
if max_diff==0:
nl.notify('Datasets appear to be identical')
return True
nl.notify('Datasets are minimally different (max_diff = %.1f)' % max_diff)
return True
def unpack_archive(fname,out_dir):
'''unpacks the archive file ``fname`` and reconstructs datasets into ``out_dir``
Datasets are reconstructed and auto-named using :meth:`create_dset`. The raw directories
that made the datasets are archive with the dataset name suffixed by ``tgz``, and any other
files found in the archive are put into ``other_files.tgz``'''
with nl.notify('Unpacking archive %s' % fname):
tmp_dir = tempfile.mkdtemp()
tmp_unpack = os.path.join(tmp_dir,'unpack')
os.makedirs(tmp_unpack)
nl.utils.unarchive(fname,tmp_unpack)
reconstruct_files(tmp_unpack)
out_dir = os.path.abspath(out_dir)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
if not os.path.exists(tmp_unpack+'-sorted'):
return
with nl.run_in(tmp_unpack+'-sorted'):
for fname in glob.glob('*.nii'):
nl.run(['gzip',fname])
for fname in glob.glob('*.nii.gz'):
new_file = os.path.join(out_dir,fname)
if not os.path.exists(new_file):
shutil.move(fname,new_file)
raw_out = os.path.join(out_dir,'raw')
if not os.path.exists(raw_out):
os.makedirs(raw_out)
for rawdir in os.listdir('.'):
rawdir_tgz = os.path.join(raw_out,rawdir+'.tgz')
if not os.path.exists(rawdir_tgz):
with tarfile.open(rawdir_tgz,'w:gz') as tgz:
tgz.add(rawdir)
if len(os.listdir(tmp_unpack))!=0:
# There are still raw files left
with tarfile.open(os.path.join(raw_out,'other_files.tgz'),'w:gz') as tgz:
tgz.add(tmp_unpack)
shutil.rmtree(tmp_dir)
def create_dset_to3d(prefix,file_list,file_order='zt',num_slices=None,num_reps=None,TR=None,slice_order='alt+z',only_dicoms=True,sort_filenames=False):
'''manually create dataset by specifying everything (not recommended, but necessary when autocreation fails)
If `num_slices` or `num_reps` is omitted, it will be inferred by the number of images. If both are omitted,
it assumes that this it not a time-dependent dataset
:only_dicoms: filter the given list by readable DICOM images
:sort_filenames: sort the given files by filename using the right-most number in the filename'''
tags = {
'num_rows': (0x0028,0x0010),
'num_reps': (0x0020,0x0105),
'TR': (0x0018,0x0080)
}
with nl.notify('Trying to create dataset %s' % prefix):
if os.path.exists(prefix):
nl.notify('Error: file "%s" already exists!' % prefix,level=nl.level.error)
return False
tagvals = {}
for f in file_list:
try:
tagvals[f] = info_for_tags(f,tags.values())
except:
pass
if only_dicoms:
new_file_list = []
for f in file_list:
if f in tagvals and len(tagvals[f][tags['num_rows']])>0:
# Only include DICOMs that actually have image information
new_file_list.append(f)
file_list = new_file_list
if sort_filenames:
def file_num(fname):
try:
nums = [x.strip('.') for x in re.findall(r'[\d.]+',fname) if x.strip('.')!='']
return float(nums[-1])
except:
return fname
file_list = sorted(file_list,key=file_num)
if len(file_list)==0:
nl.notify('Error: Couldn\'t find any valid DICOM images',level=nl.level.error)
return False
cmd = ['to3d','-skip_outliers','-quit_on_err','-prefix',prefix]
if num_slices!=None or num_reps!=None:
# Time-based dataset
if num_slices==None:
if len(file_list)%num_reps!=0:
nl.notify('Error: trying to guess # of slices, but %d (number for files) doesn\'t divide evenly into %d (number of reps)' % (len(file_list),num_reps),level=nl.level.error)
return False
num_slices = len(file_list)/num_reps
if num_reps==None:
if len(file_list)%num_slices==0:
num_reps = len(file_list)/num_slices
elif len(file_list)==1 and tags['num_reps'] in tagvals[file_list[0]]:
num_reps = tagvals[file_list[0]][tags['num_reps']]
else:
nl.notify('Error: trying to guess # of reps, but %d (number for files) doesn\'t divide evenly into %d (number of slices)' % (len(file_list),num_slices),level=nl.level.error)
return False
if TR==None:
TR = tagvals[file_list[0]][tags['TR']]
cmd += ['-time:%s'%file_order]
if file_order=='zt':
cmd += [num_slices,num_reps]
else:
cmd += [num_reps,num_slices]
cmd += [TR,slice_order]
cmd += ['-@']
cmd = [str(x) for x in cmd]
out = None
try:
p = subprocess.Popen(cmd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
out = p.communicate('\n'.join(file_list))
if p.returncode!=0:
raise Exception
except:
with nl.notify('Error: to3d returned error',level=nl.level.error):
if out:
nl.notify('stdout:\n' + out[0] + '\nstderr:\n' + out[1],level=nl.level.error)
return False
def run(command,products=None,working_directory='.',force_local=False,stderr=True,quiet=False):
'''wrapper to run external programs
:command: list containing command and parameters
(formatted the same as subprocess; must contain only strings)
:products: string or list of files that are the products of this command
if all products exist, the command will not be run, and False returned
:working_directory: will chdir to this directory
:force_local: when used with `neural.scheduler`, setting to ``True`` will disable
all job distribution functions
:stderr: forward ``stderr`` into the output
``True`` will combine ``stderr`` and ``stdout``
``False`` will return ``stdout`` and let ``stderr`` print to the console
``None`` will return ``stdout`` and suppress ``stderr``
:quiet: ``False`` (default) will print friendly messages
``True`` will suppress everything but errors
``None`` will suppress all output
Returns result in form of :class:`RunResult`
'''
with run_in(working_directory):
if products:
if isinstance(products,basestring):
products = [products]
if all([os.path.exists(x) for x in products]):
return False
command = flatten(command)
command = [str(x) for x in command]
quiet_option = False if quiet==False else True
with nl.notify('Running %s...' % command[0],level=nl.level.debug,quiet=quiet_option):
out = None
returncode = 0
try:
if stderr:
# include STDERR in STDOUT output
out = subprocess.check_output(command,stderr=subprocess.STDOUT)
elif stderr==None:
# dump STDERR into nothing
out = subprocess.check_output(command,stderr=subprocess.PIPE)
else:
# let STDERR show through to the console
out = subprocess.check_output(command)
except subprocess.CalledProcessError, e:
if quiet!=None:
nl.notify('''ERROR: %s returned a non-zero status
----COMMAND------------
%s
-----------------------
----OUTPUT-------------
%s
-----------------------
Return code: %d
''' % (command[0],' '.join(command),e.output,e.returncode),level=nl.level.error)
out = e.output
returncode = e.returncode
result = RunResult(out,returncode)
if products and returncode==0:
result.output_filename = products[0]
return result
