def get_trace(self, roi_list: List[OphysROI]) -> dc_types.ROISetDict:
"""
Extract the traces from a movie as defined by the ROIs in roi_list
Parameters
----------
roi_list -- a list of OphysROI instantiations
specifying the ROIs from which to
extract traces
Returns
-------
output -- a decrosstalk_types.ROISetDict containing the ROI and
neuropil traces associated with roi_list. For each ROI
in the ROISetDict, only the 'signal' channel will be
populated, this with the trace extracted from the movie.
"""
motion_border = [
self._motion_border['x0'], self._motion_border['x1'],
self._motion_border['y0'], self._motion_border['y1']
]
height = self.data.shape[1]
width = self.data.shape[2]
roi_mask_list = []
for roi in roi_list:
pixels = np.argwhere(roi.mask_matrix)
pixels[:, 0] += roi.y0
pixels[:, 1] += roi.x0
mask = roi_masks.create_roi_mask(width,
height,
motion_border,
pix_list=pixels[:, [1, 0]],
label=str(roi.roi_id),
mask_group=-1)
roi_mask_list.append(mask)
_traces = roi_masks.calculate_roi_and_neuropil_traces(
self.data, roi_mask_list, motion_border)
roi_traces = _traces[0]
neuropil_traces = _traces[1]
output = dc_types.ROISetDict()
for i_roi, roi in enumerate(roi_list):
trace = dc_types.ROIChannels()
trace['signal'] = roi_traces[i_roi]
output['roi'][roi.roi_id] = trace
trace = dc_types.ROIChannels()
trace['signal'] = neuropil_traces[i_roi]
output['neuropil'][roi.roi_id] = trace
return output
def test_ROIChannels():
signal = np.arange(9, dtype=float)
crosstalk = np.arange(8, 17, dtype=float)
mm = np.array([[1.2, 3.4], [5.6, 7.9]])
p_signal = signal+0.01
p_crosstalk = crosstalk+0.7
p_mm = mm+0.9
channels = dc_types.ROIChannels()
channels['signal'] = signal
channels['crosstalk'] = crosstalk
channels['mixing_matrix'] = mm
channels['poorly_converged_signal'] = p_signal
channels['poorly_converged_crosstalk'] = p_crosstalk
channels['poorly_converged_mixing_matrix'] = p_mm
channels['use_avg_mixing_matrix'] = False
np_almost(channels['signal'], signal, decimal=10)
np_almost(channels['crosstalk'], crosstalk, decimal=10)
np_almost(channels['mixing_matrix'], mm, decimal=10)
np_almost(p_signal,
channels['poorly_converged_signal'], decimal=10)
np_almost(p_crosstalk,
channels['poorly_converged_crosstalk'], decimal=10)
np_almost(p_mm,
channels['poorly_converged_mixing_matrix'], decimal=10)
assert not channels['use_avg_mixing_matrix']
def test_ROIChannel_exceptions():
bad_signals = [np.array([1, 2, 3], dtype=int),
np.array([[1.1, 2.2], [3.4, 5.4]], dtype=float),
1.4]
for bad_val in bad_signals:
channels = dc_types.ROIChannels()
with pytest.raises(ValueError):
channels['signal'] = bad_val
for bad_val in bad_signals:
channels = dc_types.ROIChannels()
with pytest.raises(ValueError):
channels['crosstalk'] = bad_val
for bad_val in bad_signals:
with pytest.raises(ValueError):
channels = dc_types.ROIChannels()
channels['poorly_converged_signal'] = bad_val
for bad_val in bad_signals:
channels = dc_types.ROIChannels()
with pytest.raises(ValueError):
channels['poorly_converged_crosstalk'] = bad_val
bad_mm = [np.array([1.5, 6.7, 3.4]),
np.array([[1, 3], [7, 9]], dtype=int),
4.5]
for bad_val in bad_mm:
channels = dc_types.ROIChannels()
with pytest.raises(ValueError):
channels['mixing_matrix'] = bad_val
for bad_val in bad_mm:
channels = dc_types.ROIChannels()
with pytest.raises(ValueError):
channels['poorly_converged_mixing_matrix'] = bad_val
channels = dc_types.ROIChannels()
with pytest.raises(ValueError):
channels['use_avg_mixing_matrix'] = 'True'
channels = dc_types.ROIChannels()
channels['signal'] = np.arange(9, dtype=float)
with pytest.raises(NotImplementedError):
channels.pop('signal')
channels = dc_types.ROIChannels()
with pytest.raises(KeyError):
channels['abracadabra'] = 9
def test_ROISetDict():
rng = np.random.RandomState(53124)
signals = list([rng.random_sample(10) for ii in range(4)])
crosstalks = list([rng.random_sample(10) for ii in range(4)])
roi_set_dict = dc_types.ROISetDict()
for ii in range(2):
c = dc_types.ROIChannels()
c['signal'] = signals[ii]
c['crosstalk'] = crosstalks[ii]
roi_set_dict['roi'][ii] = c
for ii in range(2, 4):
c = dc_types.ROIChannels()
c['signal'] = signals[ii]
c['crosstalk'] = crosstalks[ii]
roi_set_dict['neuropil'][ii-2] = c
np_almost(roi_set_dict['roi'][0]['signal'],
signals[0], decimal=10)
np_almost(roi_set_dict['roi'][1]['signal'],
signals[1], decimal=10)
np_almost(roi_set_dict['roi'][0]['crosstalk'],
crosstalks[0], decimal=10)
np_almost(roi_set_dict['roi'][1]['crosstalk'],
crosstalks[1], decimal=10)
np_almost(roi_set_dict['neuropil'][0]['signal'],
signals[2], decimal=10)
np_almost(roi_set_dict['neuropil'][1]['signal'],
signals[3], decimal=10)
np_almost(roi_set_dict['neuropil'][0]['crosstalk'],
crosstalks[2], decimal=10)
np_almost(roi_set_dict['neuropil'][1]['crosstalk'],
crosstalks[3], decimal=10)
def test_clean_negative_traces():
rng = np.random.RandomState(88123)
input_data = dc_types.ROISetDict()
trace = rng.normal(7.0, 0.2, size=1000)
trace[77] = 15.0
trace[99] = 1.0
roi = dc_types.ROIChannels()
roi['signal'] = trace
roi['crosstalk'] = rng.random_sample(1000)
input_data['roi'][0] = roi
trace = rng.normal(11.0, 0.2, size=1000)
trace[44] = 22.0
trace[88] = 1.0
roi = dc_types.ROIChannels()
roi['signal'] = trace
roi['crosstalk'] = rng.random_sample(1000)
input_data['neuropil'][0] = roi
# make sure traces with NaNs are left untouched
nan_trace = rng.normal(7.0, 0.2, size=1000)
nan_trace[11] = 17.0
nan_trace[33] = -1.0
nan_trace[44] = np.NaN
roi = dc_types.ROIChannels()
roi['signal'] = nan_trace
roi['crosstalk'] = rng.random_sample(1000)
input_data['roi'][1] = roi
cleaned = decrosstalk.clean_negative_traces(input_data)
assert cleaned['roi'][0]['signal'].min() > 6.0
assert cleaned['roi'][0]['signal'].min() < 7.0
assert abs(cleaned['roi'][0]['signal'][77] - 15.0) < 0.001
assert not np.isnan(cleaned['roi'][0]['signal']).any()
np.testing.assert_array_equal(cleaned['roi'][1]['signal'], nan_trace)
assert cleaned['neuropil'][0]['signal'].min() > 10.0
assert cleaned['neuropil'][0]['signal'].min() < 11.0
assert abs(cleaned['neuropil'][0]['signal'][44] - 22.0) < 0.001
def test_ROIDict_exceptions():
channel = dc_types.ROIChannels()
channel['signal'] = np.array([9.2, 3.4, 6.7])
channel['use_avg_mixing_matrix'] = False
roi_dict = dc_types.ROIDict()
with pytest.raises(KeyError):
roi_dict['aa'] = channel
roi_dict = dc_types.ROIDict()
with pytest.raises(ValueError):
roi_dict[11] = 'abcde'
def test_ROIDict():
channel = dc_types.ROIChannels()
channel['signal'] = np.array([9.2, 3.4, 6.7])
channel['use_avg_mixing_matrix'] = False
roi_dict = dc_types.ROIDict()
roi_dict[9] = channel
np_almost(roi_dict[9]['signal'],
np.array([9.2, 3.4, 6.7]),
decimal=9)
assert not roi_dict[9]['use_avg_mixing_matrix']
def get_raw_traces(signal_plane: DecrosstalkingOphysPlane,
ct_plane: DecrosstalkingOphysPlane) -> dc_types.ROISetDict:
"""
Get the raw signal and crosstalk traces comparing
this plane to another plane
Parameters
----------
signal_plane -- an instance of DecrosstalkingOphysPlane which will
be taken as the source of the signal
ct_plane -- another instance of DecrosstalkingOphysPlane which will
be taken as the source of the crosstalk
Returns
-------
An decrosstalk_types.ROISetDict containing the raw trace data for
the ROIs in the signal plane.
"""
signal_traces = signal_plane.movie.get_trace(signal_plane.roi_list)
crosstalk_traces = ct_plane.movie.get_trace(signal_plane.roi_list)
output = dc_types.ROISetDict()
for roi_id in signal_traces['roi'].keys():
_roi = dc_types.ROIChannels()
_neuropil = dc_types.ROIChannels()
_roi['signal'] = signal_traces['roi'][roi_id]['signal']
_roi['crosstalk'] = crosstalk_traces['roi'][roi_id]['signal']
output['roi'][roi_id] = _roi
_neuropil['signal'] = signal_traces['neuropil'][roi_id]['signal']
_neuropil['crosstalk'] = crosstalk_traces['neuropil'][roi_id]['signal']
output['neuropil'][roi_id] = _neuropil
return output
def test_ROIDict_pop_and_keys():
rng = np.random.RandomState(1123)
s1 = rng.random_sample(10)
c1 = rng.random_sample(10)
s2 = rng.random_sample(14)
c2 = rng.random_sample(14)
channel1 = dc_types.ROIChannels()
channel1['signal'] = s1
channel1['crosstalk'] = c1
channel2 = dc_types.ROIChannels()
channel2['signal'] = s2
channel2['crosstalk'] = c2
roi_dict = dc_types.ROIDict()
roi_dict[88] = channel1
roi_dict[77] = channel2
keys = roi_dict.keys()
keys.sort()
assert keys == [77, 88]
assert 77 in roi_dict
assert 88 in roi_dict
assert 55 not in roi_dict
test = roi_dict.pop(88)
assert 77 in roi_dict
assert 88 not in roi_dict
assert roi_dict.keys() == [77]
np_almost(test['signal'], s1, decimal=10)
np_almost(test['crosstalk'], c1, decimal=10)
np_almost(roi_dict[77]['signal'], s2, decimal=10)
np_almost(roi_dict[77]['crosstalk'], c2, decimal=10)
def unmix_ROI(roi_traces: dc_types.ROIChannels,
seed: int = None,
iters: int = 10) -> dc_types.ROIChannels:
"""
Unmix the signal and crosstalk traces for a single ROI
Parameters
----------
roi_traces -- a decrosstalking_types.ROIChannels containing
raw trace information for the ROI
seed -- an int used to seed the random number generator
that sklearn.decompositions.FastICA uses
iters -- an int indicating the number of iterations of
FastICA to run before giving up on convegence.
Returns
-------
A decrosstalk_types.ROIChannels containing the unmixed trace data
for the ROI
"""
ica_input = np.array([roi_traces['signal'], roi_traces['crosstalk']])
(unmixed_signals,
mixing_matrix,
roi_demixed) = ica_utils.run_ica(ica_input,
iters,
seed)
assert unmixed_signals.shape == ica_input.shape
output = dc_types.ROIChannels()
output['mixing_matrix'] = mixing_matrix
output['signal'] = unmixed_signals[0, :]
output['crosstalk'] = unmixed_signals[1, :]
output['use_avg_mixing_matrix'] = not roi_demixed
return output
def test_ROIChannels_in():
signal = np.arange(9, dtype=float)
crosstalk = np.arange(8, 17, dtype=float)
mm = np.array([[1.2, 3.4], [5.6, 7.9]])
p_signal = signal+0.01
p_crosstalk = crosstalk+0.7
p_mm = mm+0.9
channels = dc_types.ROIChannels()
channels['signal'] = signal
assert 'signal' in channels
assert 'crosstalk' not in channels
assert 'mixing_matrix' not in channels
assert 'poorly_converged_signal' not in channels
assert 'poorly_converged_crosstalk' not in channels
assert 'poorly_converged_mixing_matrix' not in channels
assert 'use_avg_mixing_matrix' not in channels
channels['crosstalk'] = crosstalk
assert 'signal' in channels
assert 'crosstalk' in channels
assert 'mixing_matrix' not in channels
assert 'poorly_converged_signal' not in channels
assert 'poorly_converged_crosstalk' not in channels
assert 'poorly_converged_mixing_matrix' not in channels
assert 'use_avg_mixing_matrix' not in channels
channels['mixing_matrix'] = mm
assert 'signal' in channels
assert 'crosstalk' in channels
assert 'mixing_matrix' in channels
assert 'poorly_converged_signal' not in channels
assert 'poorly_converged_crosstalk' not in channels
assert 'poorly_converged_mixing_matrix' not in channels
assert 'use_avg_mixing_matrix' not in channels
keys = channels.keys()
keys.sort()
assert keys == ['crosstalk', 'mixing_matrix', 'signal']
channels['poorly_converged_signal'] = p_signal
assert 'signal' in channels
assert 'crosstalk' in channels
assert 'mixing_matrix' in channels
assert 'poorly_converged_signal' in channels
assert 'poorly_converged_crosstalk' not in channels
assert 'poorly_converged_mixing_matrix' not in channels
assert 'use_avg_mixing_matrix' not in channels
channels['poorly_converged_crosstalk'] = p_crosstalk
assert 'signal' in channels
assert 'crosstalk' in channels
assert 'mixing_matrix' in channels
assert 'poorly_converged_signal' in channels
assert 'poorly_converged_crosstalk' in channels
assert 'poorly_converged_mixing_matrix' not in channels
assert 'use_avg_mixing_matrix' not in channels
keys = channels.keys()
keys.sort()
assert keys == ['crosstalk', 'mixing_matrix',
'poorly_converged_crosstalk',
'poorly_converged_signal',
'signal']
channels['poorly_converged_mixing_matrix'] = p_mm
assert 'signal' in channels
assert 'crosstalk' in channels
assert 'mixing_matrix' in channels
assert 'poorly_converged_signal' in channels
assert 'poorly_converged_crosstalk' in channels
assert 'poorly_converged_mixing_matrix' in channels
assert 'use_avg_mixing_matrix' not in channels
channels['use_avg_mixing_matrix'] = False
assert 'signal' in channels
assert 'crosstalk' in channels
assert 'mixing_matrix' in channels
assert 'poorly_converged_signal' in channels
assert 'poorly_converged_crosstalk' in channels
assert 'poorly_converged_mixing_matrix' in channels
assert 'use_avg_mixing_matrix' in channels
keys = channels.keys()
keys.sort()
assert keys == ['crosstalk',
'mixing_matrix',
'poorly_converged_crosstalk',
'poorly_converged_mixing_matrix',
'poorly_converged_signal',
'signal',
'use_avg_mixing_matrix']
def clean_negative_traces(trace_dict: dc_types.ROISetDict) -> dc_types.ROISetDict: # noqa: E501
"""
Parameters
----------
trace_dict -- a decrosstalk_types.ROISetDict containing the traces
that need to be clipped
Returns
-------
A decrosstalk_types.ROISetDict containing the clipped traces
"""
active_trace_dict = get_trace_events(trace_dict['roi'])
output_trace_dict = dc_types.ROISetDict()
roi_id_list = trace_dict['roi'].keys()
roi_id_list.sort()
for roi_id in roi_id_list:
n_t = len(trace_dict['roi'][roi_id]['signal'])
# try to select only inactive timesteps;
# if there are none, select all timesteps
# (that would be an unexpected edge case)
mask = np.ones(n_t, dtype=bool)
mask[active_trace_dict[roi_id]['signal']['events']] = False
if mask.sum() == 0:
mask[:] = True
for obj in ('roi', 'neuropil'):
# because we are running this step before culling
# traces that failed ICA, sometimes, ROIs without
# valid neuropil traces will get through
if roi_id not in trace_dict[obj]:
continue
# again: there may be traces with NaNs; these are
# going to get failed, anyway; just ignore them
# for now
if np.isnan(trace_dict[obj][roi_id]['signal']).any():
dummy = dc_types.ROIChannels()
dummy['signal'] = trace_dict[obj][roi_id]['signal']
output_trace_dict[obj][roi_id] = dummy
continue
(mean,
std) = _centered_rolling_mean(trace_dict[obj][roi_id]['signal'],
mask,
1980)
threshold = mean-2.0*std
if (threshold < 0.0).any():
msg = 'The unmixed "%s" trace for roi %d ' % (obj, roi_id)
msg += 'contained negative flux values'
logger.warning(msg)
# if there were no valid timesteps when calculating the rolling
# mean, set the threshold to a very negative value
threshold = np.where(np.logical_not(np.isnan(threshold)),
threshold,
-999.0)
if np.isnan(threshold).any():
raise RuntimeError("There were NaNs in "
"clean_negative_traces.threshold")
# clip the trace at threshold
trace = trace_dict[obj][roi_id]['signal']
trace = np.where(trace > threshold, trace, threshold)
channel = dc_types.ROIChannels()
channel['signal'] = trace
output_trace_dict[obj][roi_id] = channel
return output_trace_dict
def unmix_all_ROIs(raw_roi_traces: dc_types.ROISetDict,
seed_lookup: Dict[int, int]) -> Tuple[bool, dc_types.ROISetDict]: # noqa: E501
"""
Unmix all of the ROIs in this DecrosstalkingOphysPlane.
Parameters
----------
raw_roi_traces -- a decrosstalk_types.ROISetDict containing the
raw trace data for all the ROIs to be unmixed
seed_lookup -- a dict that maps roi_id to a seed for np.RandomState
Returns
-------
A boolean indicating whether or not we were able to successfully
unmix the ROIs in this input ROISetDict
A decrosstalk_types.ROISetDict containing the unmixed trace data for the
ROIs.
Notes
-----
This method makes two attempts at decrosstalking each ROI using
Independent Component Analysis. On the first attempt, each ROI
(not neuropil) is decrosstalked as an independent entity. It is
possible that this process will not converge for any given ROI.
On the second attempt, an average demixing matrix is constructed
from the demixing matrices of those ROIs for which the first attempt
did converge. This average demixing matrix is to decrosstalk any
ROIs for which the first attempt did not converge.
Neuropils are then decrosstalked using the demixing matrix
corresponding to their associated ROI (whether the ROI-specific
demixing matrix or, in the case that the first attempt did not
converge, the average demixing matrix).
If the first attempt does not converge for any ROIs, there are no
demixing matrices from which to construct an average demixing
matrix and it is impossible to salvage any of the ROIs. In this case,
the boolean that is the first returned object of this method will
be set to False and the output ROISetDict will be emtpy.
If decrosstalking converged for any ROIs (and an average demixing
matrix is thus possible), that boolean will be set to True.
The unmixed traces, however they were achieved, will be saved in
the output ROISetDict.
"""
output = dc_types.ROISetDict()
# first pass naively unmixing ROIs with ICA
for roi_id in raw_roi_traces['roi'].keys():
unmixed_roi = unmix_ROI(raw_roi_traces['roi'][roi_id],
seed=seed_lookup[roi_id],
iters=10)
if not unmixed_roi['use_avg_mixing_matrix']:
_out = unmixed_roi
else:
_out = dc_types.ROIChannels()
_out['use_avg_mixing_matrix'] = True
for k in unmixed_roi.keys():
if k == 'use_avg_mixing_matrix':
continue
_out['poorly_converged_%s' % k] = unmixed_roi[k]
_out[k] = np.NaN*np.zeros(unmixed_roi[k].shape, dtype=float)
output['roi'][roi_id] = _out
# calculate avg mixing matrix from successful iterations
just_roi = output['roi']
alpha_arr = np.array([min(just_roi[roi_id]['mixing_matrix'][0, 0],
just_roi[roi_id]['mixing_matrix'][0, 1])
for roi_id in just_roi.keys()
if not just_roi[roi_id]['use_avg_mixing_matrix']])
beta_arr = np.array([min(just_roi[roi_id]['mixing_matrix'][1, 0],
just_roi[roi_id]['mixing_matrix'][1, 1])
for roi_id in just_roi.keys()
if not just_roi[roi_id]['use_avg_mixing_matrix']])
assert alpha_arr.shape == beta_arr.shape
if len(alpha_arr) == 0:
return False, output
mean_alpha = alpha_arr.mean()
mean_beta = beta_arr.mean()
mean_mixing_matrix = np.zeros((2, 2), dtype=float)
mean_mixing_matrix[0, 0] = 1.0-mean_alpha
mean_mixing_matrix[0, 1] = mean_alpha
mean_mixing_matrix[1, 0] = mean_beta
mean_mixing_matrix[1, 1] = 1.0-mean_beta
inv_mean_mixing_matrix = np.linalg.inv(mean_mixing_matrix)
for roi_id in raw_roi_traces['roi'].keys():
inv_mixing_matrix = None
mixing_matrix = None
if not output['roi'][roi_id]['use_avg_mixing_matrix']:
mixing_matrix = output['roi'][roi_id]['mixing_matrix']
inv_mixing_matrix = np.linalg.inv(mixing_matrix)
else:
mixing_matrix = mean_mixing_matrix
inv_mixing_matrix = inv_mean_mixing_matrix
# assign missing outputs to ROIs that failed to converge
output['roi'][roi_id]['mixing_matrix'] = mixing_matrix
_roi_traces = raw_roi_traces['roi'][roi_id]
unmixed_signals = np.dot(inv_mixing_matrix,
np.array([_roi_traces['signal'],
_roi_traces['crosstalk']]))
output['roi'][roi_id]['signal'] = unmixed_signals[0, :]
output['roi'][roi_id]['crosstalk'] = unmixed_signals[1, :]
# assign outputs to 'neuropils'
_np_traces = raw_roi_traces['neuropil'][roi_id]
unmixed_signals = np.dot(inv_mixing_matrix,
np.array([_np_traces['signal'],
_np_traces['crosstalk']]))
neuropil = dc_types.ROIChannels()
neuropil['signal'] = unmixed_signals[0, :]
neuropil['crosstalk'] = unmixed_signals[1, :]
output['neuropil'][roi_id] = neuropil
return True, output