def _get_sorteddict(self, object, dictwithhash=False):
if isinstance(object, dict):
out = {}
for key, val in sorted(object.items()):
if isdefined(val):
out[key] = self._get_sorteddict(val, dictwithhash)
elif isinstance(object, (list,tuple)):
out = []
for val in object:
if isdefined(val):
out.append(self._get_sorteddict(val, dictwithhash))
if isinstance(object, tuple):
out = tuple(out)
else:
if isdefined(object):
if isinstance(object, str) and os.path.isfile(object):
if config.get('execution', 'hash_method').lower() == 'timestamp':
hash = hash_timestamp(object)
elif config.get('execution', 'hash_method').lower() == 'content':
hash = hash_infile(object)
else:
raise Exception("Unknown hash method: %s" % config.get('execution', 'hash_method'))
if dictwithhash:
out = (object, hash)
else:
out = hash
elif isinstance(object, float):
out = '%.10f'%object
else:
out = object
return out
def _get_sorteddict(self, object, dictwithhash=False, hash_method=None, hash_files=True):
if isinstance(object, dict):
out = {}
for key, val in sorted(object.items()):
if isdefined(val):
out[key] = self._get_sorteddict(val, dictwithhash, hash_method=hash_method, hash_files=hash_files)
elif isinstance(object, (list, tuple)):
out = []
for val in object:
if isdefined(val):
out.append(self._get_sorteddict(val, dictwithhash, hash_method=hash_method, hash_files=hash_files))
if isinstance(object, tuple):
out = tuple(out)
else:
if isdefined(object):
if hash_files and isinstance(object, str) and os.path.isfile(object):
if hash_method == None:
hash_method = config.get("execution", "hash_method")
if hash_method.lower() == "timestamp":
hash = hash_timestamp(object)
elif hash_method.lower() == "content":
hash = hash_infile(object)
else:
raise Exception("Unknown hash method: %s" % hash_method)
if dictwithhash:
out = (object, hash)
else:
out = hash
elif isinstance(object, float):
out = "%.10f" % object
else:
out = object
return out
def run(self, **inputs):
"""Execute this interface.
This interface will not raise an exception if runtime.returncode is
non-zero.
Parameters
----------
inputs : allows the interface settings to be updated
Returns
-------
results : an InterfaceResult object containing a copy of the instance
that was executed, provenance information and, if successful, results
"""
self.inputs.set(**inputs)
self._check_mandatory_inputs()
interface = self.__class__
# initialize provenance tracking
env = deepcopy(os.environ.data)
runtime = Bunch(cwd=os.getcwd(),
returncode=None,
duration=None,
environ=env,
hostname=gethostname())
t = time()
try:
runtime = self._run_interface(runtime)
runtime.duration = time() - t
results = InterfaceResult(interface, runtime,
inputs=self.inputs.get_traitsfree())
results.outputs = self.aggregate_outputs(results.runtime)
except Exception, e:
if len(e.args) == 0:
e.args = ("")
message = "\nInterface %s failed to run." % self.__class__.__name__
if config.has_option('logging', 'interface_level') and config.get('logging', 'interface_level').lower() == 'debug':
inputs_str = "Inputs:" + str(self.inputs) + "\n"
else:
inputs_str = ''
if len(e.args) == 1 and isinstance(e.args[0], str):
e.args = (e.args[0] + " ".join([message, inputs_str]),)
else:
e.args += (message, )
if inputs_str != '':
e.args += (inputs_str, )
#exception raising inhibition for special cases
if hasattr(self.inputs, 'ignore_exception') and \
isdefined(self.inputs.ignore_exception) and \
self.inputs.ignore_exception:
import traceback
runtime.traceback = traceback.format_exc()
runtime.traceback_args = e.args
return InterfaceResult(interface, runtime)
else:
raise
def _hash_infile(self, adict, key):
""" Inject file hashes into adict[key]"""
stuff = adict[key]
if not is_container(stuff):
stuff = [stuff]
file_list = []
for afile in stuff:
if is_container(afile):
hashlist = self._hash_infile({"infiles": afile}, "infiles")
hash = [val[1] for val in hashlist]
else:
if config.get("execution", "hash_method").lower() == "timestamp":
hash = hash_timestamp(afile)
elif config.get("execution", "hash_method").lower() == "content":
hash = hash_infile(afile)
else:
raise Exception("Unknown hash method: %s" % config.get("execution", "hash_method"))
file_list.append((afile, hash))
return file_list
def test_TraitedSpec_withFile():
tmp_infile = setup_file()
tmpd, nme = os.path.split(tmp_infile)
yield assert_true, os.path.exists(tmp_infile)
class spec2(nib.TraitedSpec):
moo = nib.File(exists=True)
doo = nib.traits.List(nib.File(exists=True))
infields = spec2(moo=tmp_infile,doo=[tmp_infile])
if config.get('execution', 'hash_method').lower() == 'content':
yield assert_equal, infields.hashval[1], '8c227fb727c32e00cd816c31d8fea9b9'
teardown_file(tmpd)
def __init__(self, command=None, **inputs):
super(CommandLine, self).__init__(**inputs)
self._environ = None
if not hasattr(self, "_cmd"):
self._cmd = None
if self.cmd is None and command is None:
raise Exception("Missing command")
if command:
self._cmd = command
try:
display_var = config.get("execution", "display_variable")
self.inputs.environ["DISPLAY"] = display_var
except NoOptionError:
pass
def test_caching():
temp_dir = mkdtemp(prefix='test_memory_')
old_rerun = config.get('execution', 'stop_on_first_rerun')
try:
# Prevent rerun to check that evaluation is computed only once
config.set('execution', 'stop_on_first_rerun', 'true')
mem = Memory(temp_dir)
first_nb_run = nb_runs
results = mem.cache(SideEffectInterface)(input1=2, input2=1)
assert_equal(nb_runs, first_nb_run + 1)
assert_equal(results.outputs.output1, [1, 2])
results = mem.cache(SideEffectInterface)(input1=2, input2=1)
# Check that the node hasn't been rerun
assert_equal(nb_runs, first_nb_run + 1)
assert_equal(results.outputs.output1, [1, 2])
results = mem.cache(SideEffectInterface)(input1=1, input2=1)
# Check that the node hasn been rerun
assert_equal(nb_runs, first_nb_run + 2)
assert_equal(results.outputs.output1, [1, 1])
finally:
rmtree(temp_dir)
config.set('execution', 'stop_on_first_rerun', old_rerun)
from nipype.utils.config import config
import matplotlib
matplotlib.use(config.get("execution", "matplotlib_backend"))
from vis import Overlay, PsMerge, Ps2Pdf, PlotRealignemntParameters
from threshold import ThresholdGGMM, CreateTopoFDRwithGGMM, ThresholdGMM, ThresholdFDR
from simgen import SimulationGenerator
from resampling import CalculateNonParametricFWEThreshold, CalculateProbabilityFromSamples, CalculateFDRQMap
from bootstrapping import BootstrapTimeSeries, PermuteTimeSeries
from bedpostx_particle_reader import Particle2Trackvis
from annotate_tracks import AnnotateTracts
from icc import ICC
import numpy as np
def estimate_fdr_and_fnr(true_pattern, exp_result):
false_positives = sum(exp_result[true_pattern != 1] != 0)
false_negatives = sum(exp_result[true_pattern != 0] == 0)
all_positives = np.sum(exp_result != 0)
all_negatives = np.sum(exp_result == 0)
if all_positives == 0:
fdr = 0
else:
fdr = float(false_positives)/float(all_positives)
if all_negatives == 0:
fnr = 0
else:
fnr = float(false_negatives)/float(all_negatives)
return (fdr, fnr)
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
import logging
import logging.handlers
import os
from nipype.utils.config import config
import sys
#Sets up logging for pipeline and nodewrapper execution
LOG_FILENAME = os.path.join(config.get('logging','log_directory'),
'pypeline.log')
logging.basicConfig(stream=sys.stdout)
logger = logging.getLogger('workflow')
fmlogger = logging.getLogger('filemanip')
iflogger = logging.getLogger('interface')
hdlr = logging.handlers.RotatingFileHandler(LOG_FILENAME,
maxBytes=int(config.get('logging',
'log_size')),
backupCount=int(config.get('logging',
'log_rotate')))
formatter = logging.Formatter(fmt='%(asctime)s,%(msecs)d %(name)-2s '\
'%(levelname)-2s:\n\t %(message)s',
datefmt='%y%m%d-%H:%M:%S')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.getLevelName(config.get('logging','workflow_level')))
fmlogger.addHandler(hdlr)
fmlogger.setLevel(logging.getLevelName(config.get('logging','filemanip_level')))
iflogger.addHandler(hdlr)
def copyfile(originalfile, newfile, copy=False, create_new=False):
"""Copy or symlink ``originalfile`` to ``newfile``.
Parameters
----------
originalfile : str
full path to original file
newfile : str
full path to new file
copy : Bool
specifies whether to copy or symlink files
(default=False) but only for POSIX systems
Returns
-------
None
"""
newhash = None
orighash = None
fmlogger.debug(newfile)
if create_new:
while os.path.exists(newfile):
base, fname, ext = split_filename(newfile)
s = re.search('_c[0-9]{4,4}$',fname)
i = 0
if s:
i = int(s.group()[2:])+1
fname = fname[:-6] + "_c%04d"%i
else:
fname += "_c%04d"%i
newfile = base + os.sep + fname + ext
elif os.path.exists(newfile):
if config.get('execution', 'hash_method').lower() == 'timestamp':
newhash = hash_timestamp(newfile)
elif config.get('execution', 'hash_method').lower() == 'content':
newhash = hash_infile(newfile)
fmlogger.debug("File: %s already exists,%s, copy:%d" \
% (newfile, newhash, copy))
#the following seems unnecessary
#if os.name is 'posix' and copy:
# if os.path.lexists(newfile) and os.path.islink(newfile):
# os.unlink(newfile)
# newhash = None
if os.name is 'posix' and not copy:
if os.path.lexists(newfile):
if config.get('execution', 'hash_method').lower() == 'timestamp':
orighash = hash_timestamp(originalfile)
elif config.get('execution', 'hash_method').lower() == 'content':
orighash = hash_infile(originalfile)
fmlogger.debug('Original hash: %s, %s'%(originalfile, orighash))
if newhash != orighash:
os.unlink(newfile)
if (newhash is None) or (newhash != orighash):
os.symlink(originalfile,newfile)
else:
if newhash:
if config.get('execution', 'hash_method').lower() == 'timestamp':
orighash = hash_timestamp(originalfile)
elif config.get('execution', 'hash_method').lower() == 'content':
orighash = hash_infile(originalfile)
if (newhash is None) or (newhash != orighash):
try:
fmlogger.debug("Copying File: %s->%s" \
% (newfile, originalfile))
shutil.copyfile(originalfile, newfile)
except shutil.Error, e:
fmlogger.warn(e.message)
else:
def _gen_regress(self, i_onsets, i_durations, i_amplitudes, nscans):
"""Generates a regressor for a sparse/clustered-sparse acquisition
"""
bplot = False
if isdefined(self.inputs.save_plot) and self.inputs.save_plot:
bplot=True
import matplotlib
matplotlib.use(config.get("execution", "matplotlib_backend"))
import matplotlib.pyplot as plt
TR = np.round(self.inputs.time_repetition*1000) # in ms
if self.inputs.time_acquisition:
TA = np.round(self.inputs.time_acquisition*1000) # in ms
else:
TA = TR # in ms
nvol = self.inputs.volumes_in_cluster
SCANONSET = np.round(self.inputs.scan_onset*1000)
total_time = TR*(nscans-nvol)/nvol + TA*nvol + SCANONSET
SILENCE = TR-TA*nvol
dt = TA/10.;
durations = np.round(np.array(i_durations)*1000)
if len(durations) == 1:
durations = durations*np.ones((len(i_onsets)))
onsets = np.round(np.array(i_onsets)*1000)
dttemp = gcd(TA, gcd(SILENCE, TR))
if dt < dttemp:
if dttemp % dt != 0:
dt = gcd(dttemp, dt)
if dt < 1:
raise Exception("Time multiple less than 1 ms")
iflogger.info("Setting dt = %d ms\n" % dt)
npts = int(total_time/dt)
times = np.arange(0, total_time, dt)*1e-3
timeline = np.zeros((npts))
timeline2 = np.zeros((npts))
if isdefined(self.inputs.model_hrf) and self.inputs.model_hrf:
hrf = spm_hrf(dt*1e-3)
reg_scale = 1.0
if self.inputs.scale_regressors:
boxcar = np.zeros((50.*1e3/dt))
if self.inputs.stimuli_as_impulses:
boxcar[1.*1e3/dt] = 1.0
reg_scale = float(TA/dt)
else:
boxcar[1.*1e3/dt:2.*1e3/dt] = 1.0
if isdefined(self.inputs.model_hrf) and self.inputs.model_hrf:
response = np.convolve(boxcar, hrf)
reg_scale = 1./response.max()
iflogger.info('response sum: %.4f max: %.4f'%(response.sum(), response.max()))
iflogger.info('reg_scale: %.4f'%reg_scale)
for i, t in enumerate(onsets):
idx = int(t/dt)
if i_amplitudes:
if len(i_amplitudes)>1:
timeline2[idx] = i_amplitudes[i]
else:
timeline2[idx] = i_amplitudes[0]
else:
timeline2[idx] = 1
if bplot:
plt.subplot(4, 1, 1)
plt.plot(times, timeline2)
if not self.inputs.stimuli_as_impulses:
if durations[i] == 0:
durations[i] = TA*nvol
stimdur = np.ones((int(durations[i]/dt)))
timeline2 = np.convolve(timeline2, stimdur)[0:len(timeline2)]
timeline += timeline2
timeline2[:] = 0
if bplot:
plt.subplot(4, 1, 2)
plt.plot(times, timeline)
if isdefined(self.inputs.model_hrf) and self.inputs.model_hrf:
timeline = np.convolve(timeline, hrf)[0:len(timeline)]
if isdefined(self.inputs.use_temporal_deriv) and self.inputs.use_temporal_deriv:
#create temporal deriv
timederiv = np.concatenate(([0], np.diff(timeline)))
if bplot:
plt.subplot(4, 1, 3)
plt.plot(times, timeline)
if isdefined(self.inputs.use_temporal_deriv) and self.inputs.use_temporal_deriv:
plt.plot(times, timederiv)
# sample timeline
timeline2 = np.zeros((npts))
reg = []
regderiv = []
for i, trial in enumerate(np.arange(nscans)/nvol):
scanstart = int((SCANONSET + trial*TR + (i%nvol)*TA)/dt)
#print total_time/dt, SCANONSET, TR, TA, scanstart, trial, i%2, int(TA/dt)
scanidx = scanstart+np.arange(int(TA/dt))
timeline2[scanidx] = np.max(timeline)
reg.insert(i, np.mean(timeline[scanidx])*reg_scale)
if isdefined(self.inputs.use_temporal_deriv) and self.inputs.use_temporal_deriv:
regderiv.insert(i, np.mean(timederiv[scanidx])*reg_scale)
if isdefined(self.inputs.use_temporal_deriv) and self.inputs.use_temporal_deriv:
iflogger.info('orthoganlizing derivative w.r.t. main regressor')
regderiv = orth(reg, regderiv)
if bplot:
plt.subplot(4, 1, 3)
plt.plot(times, timeline2)
plt.subplot(4, 1, 4)
plt.bar(np.arange(len(reg)), reg, width=0.5)
plt.savefig('sparse.png')
plt.savefig('sparse.svg')
if regderiv:
return [reg, regderiv]
else:
return reg
#Sets up logging for pipeline and nodewrapper execution
LOG_FILENAME = 'pypeline.log'
logging.basicConfig()
logger = logging.getLogger('workflow')
fmlogger = logging.getLogger('filemanip')
iflogger = logging.getLogger('interface')
hdlr = logging.handlers.RotatingFileHandler(LOG_FILENAME,
maxBytes=256000,
backupCount=4)
formatter = logging.Formatter(fmt='%(asctime)s,%(msecs)d %(name)-2s '\
'%(levelname)-2s:\n\t %(message)s',
datefmt='%y%m%d-%H:%M:%S')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.getLevelName(config.get('logging','workflow_level')))
fmlogger.addHandler(hdlr)
fmlogger.setLevel(logging.getLevelName(config.get('logging','filemanip_level')))
iflogger.addHandler(hdlr)
iflogger.setLevel(logging.getLevelName(config.get('logging','interface_level')))
class WorkflowBase(object):
""" Define common attributes and functions for workflows and nodes
"""
def __init__(self, name=None, base_dir=None,
overwrite=False, **kwargs):
""" Initialize base parameters of a workflow or node
Parameters
----------
from nipype.utils.config import config
import matplotlib
matplotlib.use(config.get("execution", "matplotlib_backend"))
from vis import Overlay, PsMerge, Ps2Pdf, PlotRealignemntParameters
from threshold import ThresholdGGMM, CreateTopoFDRwithGGMM, ThresholdGMM, ThresholdFDR
from simgen import SimulationGenerator
from resampling import CalculateNonParametricFWEThreshold, CalculateProbabilityFromSamples, CalculateFDRQMap
from bootstrapping import BootstrapTimeSeries, PermuteTimeSeries
from bedpostx_particle_reader import Particle2Trackvis
from annotate_tracks import AnnotateTracts
from icc import ICC
import numpy as np
def estimate_fdr_and_fnr(true_pattern, exp_result):
false_positives = sum(exp_result[true_pattern != 1] != 0)
false_negatives = sum(exp_result[true_pattern != 0] == 0)
all_positives = np.sum(exp_result != 0)
all_negatives = np.sum(exp_result == 0)
if all_positives == 0:
fdr = 0
else:
fdr = float(false_positives) / float(all_positives)
if all_negatives == 0:
fnr = 0
else:
fnr = float(false_negatives) / float(all_negatives)
return (fdr, fnr)