def test_ROISetDict_exceptions():
roi_set = dc_types.ROISetDict()
with pytest.raises(NotImplementedError):
roi_set['roi'] = dc_types.ROIDict()
with pytest.raises(KeyError):
roi_set[9]
def get_trace(self, roi_list: List[OphysROI]) -> dc_types.ROISetDict:
"""
Extract the traces from a movie as defined by the ROIs in roi_list
Parameters
----------
roi_list -- a list of OphysROI instantiations
specifying the ROIs from which to
extract traces
Returns
-------
output -- a decrosstalk_types.ROISetDict containing the ROI and
neuropil traces associated with roi_list. For each ROI
in the ROISetDict, only the 'signal' channel will be
populated, this with the trace extracted from the movie.
"""
motion_border = [
self._motion_border['x0'], self._motion_border['x1'],
self._motion_border['y0'], self._motion_border['y1']
]
height = self.data.shape[1]
width = self.data.shape[2]
roi_mask_list = []
for roi in roi_list:
pixels = np.argwhere(roi.mask_matrix)
pixels[:, 0] += roi.y0
pixels[:, 1] += roi.x0
mask = RoiMask.create_roi_mask(width,
height,
motion_border,
pix_list=pixels[:, [1, 0]],
label=str(roi.roi_id),
mask_group=-1)
roi_mask_list.append(mask)
_traces = calculate_roi_and_neuropil_traces(self.data, roi_mask_list,
motion_border)
roi_traces = _traces[0]
neuropil_traces = _traces[1]
output = dc_types.ROISetDict()
for i_roi, roi in enumerate(roi_list):
trace = dc_types.ROIChannels()
trace['signal'] = roi_traces[i_roi]
output['roi'][roi.roi_id] = trace
trace = dc_types.ROIChannels()
trace['signal'] = neuropil_traces[i_roi]
output['neuropil'][roi.roi_id] = trace
return output
def test_clean_negative_traces():
rng = np.random.RandomState(88123)
input_data = dc_types.ROISetDict()
trace = rng.normal(7.0, 0.2, size=1000)
trace[77] = 15.0
trace[99] = 1.0
roi = dc_types.ROIChannels()
roi['signal'] = trace
roi['crosstalk'] = rng.random_sample(1000)
input_data['roi'][0] = roi
trace = rng.normal(11.0, 0.2, size=1000)
trace[44] = 22.0
trace[88] = 1.0
roi = dc_types.ROIChannels()
roi['signal'] = trace
roi['crosstalk'] = rng.random_sample(1000)
input_data['neuropil'][0] = roi
# make sure traces with NaNs are left untouched
nan_trace = rng.normal(7.0, 0.2, size=1000)
nan_trace[11] = 17.0
nan_trace[33] = -1.0
nan_trace[44] = np.NaN
roi = dc_types.ROIChannels()
roi['signal'] = nan_trace
roi['crosstalk'] = rng.random_sample(1000)
input_data['roi'][1] = roi
cleaned = decrosstalk.clean_negative_traces(input_data)
assert cleaned['roi'][0]['signal'].min() > 6.0
assert cleaned['roi'][0]['signal'].min() < 7.0
assert abs(cleaned['roi'][0]['signal'][77] - 15.0) < 0.001
assert not np.isnan(cleaned['roi'][0]['signal']).any()
np.testing.assert_array_equal(cleaned['roi'][1]['signal'], nan_trace)
assert cleaned['neuropil'][0]['signal'].min() > 10.0
assert cleaned['neuropil'][0]['signal'].min() < 11.0
assert abs(cleaned['neuropil'][0]['signal'][44] - 22.0) < 0.001
def test_ROISetDict():
rng = np.random.RandomState(53124)
signals = list([rng.random_sample(10) for ii in range(4)])
crosstalks = list([rng.random_sample(10) for ii in range(4)])
roi_set_dict = dc_types.ROISetDict()
for ii in range(2):
c = dc_types.ROIChannels()
c['signal'] = signals[ii]
c['crosstalk'] = crosstalks[ii]
roi_set_dict['roi'][ii] = c
for ii in range(2, 4):
c = dc_types.ROIChannels()
c['signal'] = signals[ii]
c['crosstalk'] = crosstalks[ii]
roi_set_dict['neuropil'][ii - 2] = c
np_almost(roi_set_dict['roi'][0]['signal'], signals[0], decimal=10)
np_almost(roi_set_dict['roi'][1]['signal'], signals[1], decimal=10)
np_almost(roi_set_dict['roi'][0]['crosstalk'], crosstalks[0], decimal=10)
np_almost(roi_set_dict['roi'][1]['crosstalk'], crosstalks[1], decimal=10)
np_almost(roi_set_dict['neuropil'][0]['signal'], signals[2], decimal=10)
np_almost(roi_set_dict['neuropil'][1]['signal'], signals[3], decimal=10)
np_almost(roi_set_dict['neuropil'][0]['crosstalk'],
crosstalks[2],
decimal=10)
np_almost(roi_set_dict['neuropil'][1]['crosstalk'],
crosstalks[3],
decimal=10)
def get_raw_traces(signal_plane: DecrosstalkingOphysPlane,
ct_plane: DecrosstalkingOphysPlane) -> dc_types.ROISetDict:
"""
Get the raw signal and crosstalk traces comparing
this plane to another plane
Parameters
----------
signal_plane -- an instance of DecrosstalkingOphysPlane which will
be taken as the source of the signal
ct_plane -- another instance of DecrosstalkingOphysPlane which will
be taken as the source of the crosstalk
Returns
-------
An decrosstalk_types.ROISetDict containing the raw trace data for
the ROIs in the signal plane.
"""
signal_traces = signal_plane.movie.get_trace(signal_plane.roi_list)
crosstalk_traces = ct_plane.movie.get_trace(signal_plane.roi_list)
output = dc_types.ROISetDict()
for roi_id in signal_traces['roi'].keys():
_roi = dc_types.ROIChannels()
_neuropil = dc_types.ROIChannels()
_roi['signal'] = signal_traces['roi'][roi_id]['signal']
_roi['crosstalk'] = crosstalk_traces['roi'][roi_id]['signal']
output['roi'][roi_id] = _roi
_neuropil['signal'] = signal_traces['neuropil'][roi_id]['signal']
_neuropil['crosstalk'] = crosstalk_traces['neuropil'][roi_id]['signal']
output['neuropil'][roi_id] = _neuropil
return output
def create_dataset():
"""
Create a test dataset exercising all possible permutations of
validity flags.
Returns
-------
dict
'roi_flags': a dict mimicing the roi_flags returned by the pipeline
'raw_traces': ROISetDict of valid traces
'invalid_raw_traces': same as above for invalid raw traces
'unmixed_traces': ROISetDict of valid unmixed traces
'invalid_unmixed_traces': same as above fore invalid unmixed traces
'raw_events': ROIEventSet of activity in valid raw traces
'invalid_raw_events': same as above for invalid raw traces
'unmixed_events': ROIEventSet of activity in valid unmixed traces
'invalid_unmixed_events': same as above for invalid unmixed traces
'true_flags': ground truth values of validity flags for all ROIs
"""
rng = np.random.RandomState(172)
n_t = 10
raw_traces = dc_types.ROISetDict()
invalid_raw_traces = dc_types.ROISetDict()
unmixed_traces = dc_types.ROISetDict()
invalid_unmixed_traces = dc_types.ROISetDict()
raw_events = dc_types.ROIEventSet()
invalid_raw_events = dc_types.ROIEventSet()
unmixed_events = dc_types.ROIEventSet()
invalid_unmixed_events = dc_types.ROIEventSet()
roi_flags = {}
roi_flags['decrosstalk_ghost'] = []
true_flags = []
iterator = itertools.product([True, False], [True, False], [True, False],
[True, False], [True, False], [True, False])
roi_id = -1
for _f in iterator:
roi_id += 1
flags = {
'valid_raw_trace': _f[0],
'valid_raw_active_trace': _f[1],
'valid_unmixed_trace': _f[2],
'valid_unmixed_active_trace': _f[3],
'converged': _f[4],
'ghost': _f[5]
}
if not flags['valid_raw_trace']:
flags['valid_raw_active_trace'] = False
flags['valid_unmixed_trace'] = False
flags['valid_unmixed_active_trace'] = False
if not flags['valid_raw_active_trace']:
flags['valid_unmixed_trace'] = False
flags['valid_unmixed_active_trace'] = False
if not flags['valid_unmixed_trace']:
flags['valid_unmixed_active_trace'] = False
true_flags.append(flags)
# raw traces
raw_roi = dc_types.ROIChannels()
raw_roi['signal'] = rng.random_sample(n_t)
raw_roi['crosstalk'] = rng.random_sample(n_t)
raw_np = dc_types.ROIChannels()
raw_np['signal'] = rng.random_sample(n_t)
raw_np['crosstalk'] = rng.random_sample(n_t)
if flags['valid_raw_trace'] and flags['valid_raw_active_trace']:
raw_traces['roi'][roi_id] = raw_roi
raw_traces['neuropil'][roi_id] = raw_np
else:
invalid_raw_traces['roi'][roi_id] = raw_roi
invalid_raw_traces['neuropil'][roi_id] = raw_np
if not flags['valid_raw_trace']:
continue
# raw trace events
ee = dc_types.ROIEventChannels()
e = dc_types.ROIEvents()
e['events'] = rng.choice(np.arange(n_t, dtype=int), 3)
e['trace'] = rng.random_sample(3)
ee['signal'] = e
e = dc_types.ROIEvents()
e['events'] = rng.choice(np.arange(n_t, dtype=int), 3)
e['trace'] = rng.random_sample(3)
ee['crosstalk'] = e
if flags['valid_raw_active_trace']:
raw_events[roi_id] = ee
else:
invalid_raw_events[roi_id] = ee
continue
# unmixed traces
unmixed_roi = dc_types.ROIChannels()
unmixed_roi['signal'] = rng.random_sample(n_t)
unmixed_roi['crosstalk'] = rng.random_sample(n_t)
unmixed_roi['mixing_matrix'] = rng.random_sample((2, 2))
unmixed_roi['use_avg_mixing_matrix'] = not flags['converged']
if not flags['converged']:
unmixed_roi['poorly_converged_signal'] = rng.random_sample(n_t)
unmixed_roi['poorly_converged_crosstalk'] = rng.random_sample(n_t)
mm = rng.random_sample((2, 2))
unmixed_roi['poorly_converged_mixing_matrix'] = mm
unmixed_np = dc_types.ROIChannels()
unmixed_np['signal'] = rng.random_sample(n_t)
unmixed_np['crosstalk'] = rng.random_sample(n_t)
unmixed_np['mixing_matrix'] = rng.random_sample((2, 2))
unmixed_np['use_avg_mixing_matrix'] = not flags['converged']
if not flags['converged']:
unmixed_np['poorly_converged_signal'] = rng.random_sample(n_t)
unmixed_np['poorly_converged_crosstalk'] = rng.random_sample(n_t)
mm = rng.random_sample((2, 2))
unmixed_np['poorly_converged_mixing_matrix'] = mm
is_valid = (flags['valid_unmixed_trace']
and flags['valid_unmixed_active_trace'])
if is_valid:
unmixed_traces['roi'][roi_id] = unmixed_roi
unmixed_traces['neuropil'][roi_id] = unmixed_np
else:
invalid_unmixed_traces['roi'][roi_id] = unmixed_roi
invalid_unmixed_traces['neuropil'][roi_id] = unmixed_np
if not flags['valid_unmixed_trace']:
continue
# unmixedtrace events
ee = dc_types.ROIEventChannels()
e = dc_types.ROIEvents()
e['events'] = rng.choice(np.arange(n_t, dtype=int), 3)
e['trace'] = rng.random_sample(3)
ee['signal'] = e
e = dc_types.ROIEvents()
e['events'] = rng.choice(np.arange(n_t, dtype=int), 3)
e['trace'] = rng.random_sample(3)
ee['crosstalk'] = e
if flags['valid_unmixed_active_trace']:
unmixed_events[roi_id] = ee
else:
invalid_unmixed_events[roi_id] = ee
continue
if flags['ghost']:
roi_flags['decrosstalk_ghost'].append(roi_id)
output = {}
output['roi_flags'] = roi_flags
output['raw_traces'] = raw_traces
output['invalid_raw_traces'] = invalid_raw_traces
output['unmixed_traces'] = unmixed_traces
output['invalid_unmixed_traces'] = invalid_unmixed_traces
output['raw_events'] = raw_events
output['invalid_raw_events'] = invalid_raw_events
output['unmixed_events'] = unmixed_events
output['invalid_unmixed_events'] = invalid_unmixed_events
output['true_flags'] = true_flags
return output
def unmix_all_ROIs(
raw_roi_traces: dc_types.ROISetDict,
seed_lookup: Dict[int, int]) -> Tuple[bool, dc_types.ROISetDict]:
"""
Unmix all of the ROIs in this DecrosstalkingOphysPlane.
Parameters
----------
raw_roi_traces -- a decrosstalk_types.ROISetDict containing the
raw trace data for all the ROIs to be unmixed
seed_lookup -- a dict that maps roi_id to a seed for np.RandomState
Returns
-------
A boolean indicating whether or not we were able to successfully
unmix the ROIs in this input ROISetDict
A decrosstalk_types.ROISetDict containing the unmixed trace data for the
ROIs.
Notes
-----
This method makes two attempts at decrosstalking each ROI using
Independent Component Analysis. On the first attempt, each ROI
(not neuropil) is decrosstalked as an independent entity. It is
possible that this process will not converge for any given ROI.
On the second attempt, an average demixing matrix is constructed
from the demixing matrices of those ROIs for which the first attempt
did converge. This average demixing matrix is to decrosstalk any
ROIs for which the first attempt did not converge.
Neuropils are then decrosstalked using the demixing matrix
corresponding to their associated ROI (whether the ROI-specific
demixing matrix or, in the case that the first attempt did not
converge, the average demixing matrix).
If the first attempt does not converge for any ROIs, there are no
demixing matrices from which to construct an average demixing
matrix and it is impossible to salvage any of the ROIs. In this case,
the boolean that is the first returned object of this method will
be set to False and the output ROISetDict will be emtpy.
If decrosstalking converged for any ROIs (and an average demixing
matrix is thus possible), that boolean will be set to True.
The unmixed traces, however they were achieved, will be saved in
the output ROISetDict.
"""
output = dc_types.ROISetDict()
# first pass naively unmixing ROIs with ICA
for roi_id in raw_roi_traces['roi'].keys():
unmixed_roi = unmix_ROI(raw_roi_traces['roi'][roi_id],
seed=seed_lookup[roi_id],
iters=10)
if not unmixed_roi['use_avg_mixing_matrix']:
_out = unmixed_roi
else:
_out = dc_types.ROIChannels()
_out['use_avg_mixing_matrix'] = True
for k in unmixed_roi.keys():
if k == 'use_avg_mixing_matrix':
continue
_out['poorly_converged_%s' % k] = unmixed_roi[k]
_out[k] = np.NaN * np.zeros(unmixed_roi[k].shape, dtype=float)
output['roi'][roi_id] = _out
# calculate avg mixing matrix from successful iterations
just_roi = output['roi']
alpha_arr = np.array([
min(just_roi[roi_id]['mixing_matrix'][0, 0],
just_roi[roi_id]['mixing_matrix'][0, 1])
for roi_id in just_roi.keys()
if not just_roi[roi_id]['use_avg_mixing_matrix']
])
beta_arr = np.array([
min(just_roi[roi_id]['mixing_matrix'][1, 0],
just_roi[roi_id]['mixing_matrix'][1, 1])
for roi_id in just_roi.keys()
if not just_roi[roi_id]['use_avg_mixing_matrix']
])
assert alpha_arr.shape == beta_arr.shape
if len(alpha_arr) == 0:
return False, output
mean_alpha = alpha_arr.mean()
mean_beta = beta_arr.mean()
mean_mixing_matrix = np.zeros((2, 2), dtype=float)
mean_mixing_matrix[0, 0] = 1.0 - mean_alpha
mean_mixing_matrix[0, 1] = mean_alpha
mean_mixing_matrix[1, 0] = mean_beta
mean_mixing_matrix[1, 1] = 1.0 - mean_beta
inv_mean_mixing_matrix = np.linalg.inv(mean_mixing_matrix)
for roi_id in raw_roi_traces['roi'].keys():
inv_mixing_matrix = None
mixing_matrix = None
if not output['roi'][roi_id]['use_avg_mixing_matrix']:
mixing_matrix = output['roi'][roi_id]['mixing_matrix']
inv_mixing_matrix = np.linalg.inv(mixing_matrix)
else:
mixing_matrix = mean_mixing_matrix
inv_mixing_matrix = inv_mean_mixing_matrix
# assign missing outputs to ROIs that failed to converge
output['roi'][roi_id]['mixing_matrix'] = mixing_matrix
_roi_traces = raw_roi_traces['roi'][roi_id]
unmixed_signals = np.dot(
inv_mixing_matrix,
np.array([_roi_traces['signal'], _roi_traces['crosstalk']]))
output['roi'][roi_id]['signal'] = unmixed_signals[0, :]
output['roi'][roi_id]['crosstalk'] = unmixed_signals[1, :]
# assign outputs to 'neuropils'
_np_traces = raw_roi_traces['neuropil'][roi_id]
unmixed_signals = np.dot(
inv_mixing_matrix,
np.array([_np_traces['signal'], _np_traces['crosstalk']]))
neuropil = dc_types.ROIChannels()
neuropil['signal'] = unmixed_signals[0, :]
neuropil['crosstalk'] = unmixed_signals[1, :]
output['neuropil'][roi_id] = neuropil
return True, output
def run_decrosstalk(
signal_plane: DecrosstalkingOphysPlane,
ct_plane: DecrosstalkingOphysPlane,
cache_dir: str = None,
clobber: bool = False,
new_style_output: bool = False
) -> Tuple[dict, Tuple[dc_types.ROISetDict, dc_types.ROISetDict], Tuple[
dc_types.ROISetDict, dc_types.ROISetDict], Tuple[
dc_types.ROIEventSet, dc_types.ROIEventSet], Tuple[
dc_types.ROIEventSet, dc_types.ROIEventSet]]:
"""
Actually run the decrosstalking pipeline, comparing two
DecrosstalkingOphysPlanes
Parameters
----------
signal_plane -- the DecrosstalkingOphysPlane characterizing the
signal plane
ct_plane -- the DecrosstalkingOphysPlane characterizing the crosstalk plane
cache_dir -- the directory in which to write the QC output
(if None, the output does not get written)
clobber -- a boolean indicating whether or not to overwrite
pre-existing output files (default: False)
new_style_output -- a boolean (default: False)
Returns
-------
roi_flags -- a dict listing the ROI IDs of ROIs that were
ruled invalid for different reasons, namely:
'decrosstalk_ghost' -- ROIs that are ghosts
'decrosstalk_invalid_raw' -- ROIs with invalid
raw traces
'decrosstalk_invalid_raw_active' -- ROIs with invalid
raw active traces
'decrosstalk_invalid_unmixed' -- ROIs with invalid
unmixed traces
'decrosstalk_invalid_unmixed_active' -- ROIs with invalid
unmixed active traces
(raw_traces,
invalid_raw_traces) -- two decrosstalk_types.ROISetDicts containing
the raw trace data for the ROIs and the
invalid raw traces
(unmixed_traces,
invalid_unmixed_traces) -- two decrosstalk_types.ROISetDicts containing
the unmixed trace data for the ROIs and then
invalid unmixed traces
(raw_trace_events,
invalid_raw_trace_events) -- two decrosstalk_types.ROIEventSets
characterizing the active timestamps
from the raw traces and the invalid
active timestamps
(unmixed_trace_events,
invalid_unmixed_trace_events) -- two decrosstalk_types.ROIEventSets
characterizing the active timestamps
from the unmixed traces and the invalid
unmixed trace events
"""
raw_traces = dc_types.ROISetDict()
unmixed_traces = dc_types.ROISetDict()
raw_trace_events = dc_types.ROIEventSet()
unmixed_trace_events = dc_types.ROIEventSet()
invalid_raw_traces = dc_types.ROISetDict()
invalid_unmixed_traces = dc_types.ROISetDict()
invalid_raw_trace_events = dc_types.ROIEventSet()
invalid_unmixed_trace_events = dc_types.ROIEventSet()
roi_flags: Dict[str, List[int]] = {}
ghost_key = 'decrosstalk_ghost'
raw_key = 'decrosstalk_invalid_raw'
raw_active_key = 'decrosstalk_invalid_raw_active'
unmixed_key = 'decrosstalk_invalid_unmixed'
unmixed_active_key = 'decrosstalk_invalid_unmixed_active'
roi_flags[ghost_key] = []
roi_flags[raw_key] = []
roi_flags[unmixed_key] = []
roi_flags[raw_active_key] = []
roi_flags[unmixed_active_key] = []
# If there are no ROIs in the signal plane,
# just return a set of empty outputs
if len(signal_plane.roi_list) == 0:
return (roi_flags, (raw_traces, invalid_raw_traces),
(unmixed_traces, invalid_unmixed_traces),
(raw_trace_events, invalid_raw_trace_events),
(unmixed_trace_events, invalid_unmixed_trace_events))
###############################
# extract raw traces
raw_traces = get_raw_traces(signal_plane, ct_plane)
raw_trace_validation = d_utils.validate_traces(raw_traces)
# remove invalid raw traces
invalid_raw_trace_roi_id = []
for roi_id in raw_trace_validation:
if not raw_trace_validation[roi_id]:
invalid_raw_trace_roi_id.append(roi_id)
_roi = raw_traces['roi'].pop(roi_id)
_neuropil = raw_traces['neuropil'].pop(roi_id)
invalid_raw_traces['roi'][roi_id] = _roi
invalid_raw_traces['neuropil'][roi_id] = _neuropil
roi_flags[raw_key] += invalid_raw_trace_roi_id
if len(raw_traces['roi']) == 0:
msg = 'No raw traces were valid when applying '
msg += 'decrosstalk to ophys_experiment_id: '
msg += '%d (%d)' % (signal_plane.experiment_id, ct_plane.experiment_id)
logger.error(msg)
return (roi_flags, (raw_traces, invalid_raw_traces),
(unmixed_traces, invalid_unmixed_traces),
(raw_trace_events, invalid_raw_trace_events),
(unmixed_trace_events, invalid_unmixed_trace_events))
#########################################
# detect activity in raw traces
raw_trace_events = get_trace_events(raw_traces['roi'])
# For each ROI, calculate a random seed based on the flux
# in all timestamps *not* chosen as events (presumably,
# random noise)
roi_to_seed = {}
two_to_32 = 2**32
for roi_id in raw_trace_events.keys():
flux_mask = np.ones(len(raw_traces['roi'][roi_id]['signal']),
dtype=bool)
if len(raw_trace_events[roi_id]['signal']['events']) > 0:
flux_mask[raw_trace_events[roi_id]['signal']['events']] = False
_flux = np.abs(raw_traces['roi'][roi_id]['signal'][flux_mask])
flux_sum = np.round(_flux.sum()).astype(int)
roi_to_seed[roi_id] = flux_sum % two_to_32
# remove ROIs with invalid active raw traces
roi_id_list = list(raw_trace_events.keys())
for roi_id in roi_id_list:
signal = raw_trace_events[roi_id]['signal']['trace']
if len(signal) == 0 or np.isnan(signal).any():
roi_flags[raw_active_key].append(roi_id)
_events = raw_trace_events.pop(roi_id)
_roi = raw_traces['roi'].pop(roi_id)
_neuropil = raw_traces['neuropil'].pop(roi_id)
invalid_raw_traces['roi'][roi_id] = _roi
invalid_raw_traces['neuropil'][roi_id] = _neuropil
invalid_raw_trace_events[roi_id] = _events
# if there was no activity in the raw traces, return an
# empty ROISetDict because none of the ROIs were valid
if len(raw_traces['roi']) == 0:
return (roi_flags, (raw_traces, invalid_raw_traces),
(unmixed_traces, invalid_unmixed_traces),
(raw_trace_events, invalid_raw_trace_events),
(unmixed_trace_events, invalid_unmixed_trace_events))
###########################################################
# use Independent Component Analysis to separate out signal
# and crosstalk
(ica_converged, unmixed_traces) = unmix_all_ROIs(raw_traces, roi_to_seed)
# clip dips in signal channel
clipped_traces = clean_negative_traces(unmixed_traces)
# save old signal to 'unclipped_signal'
# save new signal to 'signal'
for obj in ('roi', 'neuropil'):
for roi_id in unmixed_traces[obj].keys():
s = unmixed_traces[obj][roi_id]['signal']
unmixed_traces[obj][roi_id]['unclipped_signal'] = s
s = clipped_traces[obj][roi_id]['signal']
unmixed_traces[obj][roi_id]['signal'] = s
if not ica_converged:
for roi_id in unmixed_traces['roi'].keys():
roi_flags[unmixed_key].append(roi_id)
msg = 'ICA did not converge for any ROIs when '
msg += 'applying decrosstalk to ophys_experiment_id: '
msg += '%d (%d)' % (signal_plane.experiment_id, ct_plane.experiment_id)
logger.error(msg)
return (roi_flags, (raw_traces, invalid_raw_traces),
(unmixed_traces, invalid_unmixed_traces),
(raw_trace_events, invalid_raw_trace_events),
(unmixed_trace_events, invalid_unmixed_trace_events))
unmixed_trace_validation = d_utils.validate_traces(unmixed_traces)
# remove invalid unmixed traces
invalid_unmixed_trace_roi_id = []
for roi_id in unmixed_trace_validation:
if not unmixed_trace_validation[roi_id]:
invalid_unmixed_trace_roi_id.append(roi_id)
_roi = unmixed_traces['roi'].pop(roi_id)
_neuropil = unmixed_traces['neuropil'].pop(roi_id)
invalid_unmixed_traces['roi'][roi_id] = _roi
invalid_unmixed_traces['neuropil'][roi_id] = _neuropil
roi_flags[unmixed_key] += invalid_unmixed_trace_roi_id
if len(unmixed_traces['roi']) == 0:
msg = 'No unmixed traces were valid when applying '
msg += 'decrosstalk to ophys_experiment_id: '
msg += '%d (%d)' % (signal_plane.experiment_id, ct_plane.experiment_id)
logger.error(msg)
return (roi_flags, (raw_traces, invalid_raw_traces),
(unmixed_traces, invalid_unmixed_traces),
(raw_trace_events, invalid_raw_trace_events),
(unmixed_trace_events, invalid_unmixed_trace_events))
###################################################
# Detect activity in unmixed traces
unmixed_trace_events = get_trace_events(unmixed_traces['roi'])
# Sometimes, unmixed_trace_events will return an array of NaNs.
# Until we can debug that behavior, we will log those errors,
# store the relevaten ROIs as decrosstalk_invalid_unmixed_trace,
# and cull those ROIs from the data
invalid_active_trace: Dict[str, List[int]] = {}
invalid_active_trace['signal'] = []
invalid_active_trace['crosstalk'] = []
active_trace_had_NaNs = False
for roi_id in unmixed_trace_events.keys():
local_traces = unmixed_trace_events[roi_id]
for channel in ('signal', 'crosstalk'):
is_valid = True
if len(local_traces[channel]['trace']) == 0:
is_valid = False
else:
nan_trace = np.isnan(local_traces[channel]['trace']).any()
nan_events = np.isnan(local_traces[channel]['events']).any()
if nan_trace or nan_events:
is_valid = False
active_trace_had_NaNs = True
if not is_valid:
invalid_active_trace[channel].append(roi_id)
if active_trace_had_NaNs:
msg = 'ophys_experiment_id: %d (%d) ' % (signal_plane.experiment_id,
ct_plane.experiment_id)
msg += 'had ROIs with active event channels that contained NaNs'
logger.error(msg)
# remove ROIs with empty or NaN active trace signal channels
# from the data being processed
for roi_id in invalid_active_trace['signal']:
roi_flags[unmixed_active_key].append(roi_id)
_events = unmixed_trace_events.pop(roi_id)
_roi = unmixed_traces['roi'].pop(roi_id)
_neuropil = unmixed_traces['neuropil'].pop(roi_id)
invalid_unmixed_traces['roi'][roi_id] = _roi
invalid_unmixed_traces['neuropil'][roi_id] = _neuropil
invalid_unmixed_trace_events[roi_id] = _events
########################################################
# For each ROI, assess whether or not it is a "ghost"
# (i.e. whether any of its activity is due to the signal,
# independent of the crosstalk; if not, it is a ghost)
independent_events = {}
ghost_roi_id = []
for roi_id in unmixed_trace_events.keys():
signal = unmixed_trace_events[roi_id]['signal']
crosstalk = unmixed_trace_events[roi_id]['crosstalk']
(is_a_cell,
ind_events) = d_utils.validate_cell_crosstalk(signal, crosstalk)
local = {'is_a_cell': is_a_cell, 'independent_events': ind_events}
independent_events[roi_id] = local
if not is_a_cell:
ghost_roi_id.append(roi_id)
roi_flags[ghost_key] += ghost_roi_id
return (roi_flags, (raw_traces, invalid_raw_traces),
(unmixed_traces, invalid_unmixed_traces),
(raw_trace_events, invalid_raw_trace_events),
(unmixed_trace_events, invalid_unmixed_trace_events))
def clean_negative_traces(
trace_dict: dc_types.ROISetDict) -> dc_types.ROISetDict:
"""
Parameters
----------
trace_dict -- a decrosstalk_types.ROISetDict containing the traces
that need to be clipped
Returns
-------
A decrosstalk_types.ROISetDict containing the clipped traces
"""
active_trace_dict = get_trace_events(trace_dict['roi'])
output_trace_dict = dc_types.ROISetDict()
roi_id_list = trace_dict['roi'].keys()
roi_id_list.sort()
for roi_id in roi_id_list:
n_t = len(trace_dict['roi'][roi_id]['signal'])
# try to select only inactive timesteps;
# if there are none, select all timesteps
# (that would be an unexpected edge case)
mask = np.ones(n_t, dtype=bool)
mask[active_trace_dict[roi_id]['signal']['events']] = False
if mask.sum() == 0:
mask[:] = True
for obj in ('roi', 'neuropil'):
# because we are running this step before culling
# traces that failed ICA, sometimes, ROIs without
# valid neuropil traces will get through
if roi_id not in trace_dict[obj]:
continue
# again: there may be traces with NaNs; these are
# going to get failed, anyway; just ignore them
# for now
if np.isnan(trace_dict[obj][roi_id]['signal']).any():
dummy = dc_types.ROIChannels()
dummy['signal'] = trace_dict[obj][roi_id]['signal']
output_trace_dict[obj][roi_id] = dummy
continue
(mean,
std) = _centered_rolling_mean(trace_dict[obj][roi_id]['signal'],
mask, 1980)
threshold = mean - 2.0 * std
if (threshold < 0.0).any():
msg = 'The unmixed "%s" trace for roi %d ' % (obj, roi_id)
msg += 'contained negative flux values'
logger.warning(msg)
# if there were no valid timesteps when calculating the rolling
# mean, set the threshold to a very negative value
threshold = np.where(np.logical_not(np.isnan(threshold)),
threshold, -999.0)
if np.isnan(threshold).any():
raise RuntimeError("There were NaNs in "
"clean_negative_traces.threshold")
# clip the trace at threshold
trace = trace_dict[obj][roi_id]['signal']
trace = np.where(trace > threshold, trace, threshold)
channel = dc_types.ROIChannels()
channel['signal'] = trace
output_trace_dict[obj][roi_id] = channel
return output_trace_dict