Npeaks = 5
traces = cnm.C + cnm.YrA
# TODO: todocument
idx_components, idx_components_bad = cm.components_evaluation.estimate_components_quality(
traces, Yr, cnm.A, cnm.C, cnm.b, cnm.f, final_frate=final_frate, Npeaks=Npeaks,
r_values_min=r_values_min, fitness_min=fitness_min, fitness_delta_min=fitness_delta_min, dview=dview)
print(('Keeping ' + str(len(idx_components)) +
' and discarding ' + str(len(idx_components_bad))))
#%%
A_, C_, YrA_, b_, f_ = (cnm.A[:, idx_components], cnm.C[idx_components],
cnm.YrA[idx_components], cnm.b, cnm.f)
#%%
YrA_GT = compute_residuals(np.array(Yr) - b0, A, b, C, f, dview=None)
cm.utils.visualization.view_patches_bar(Yr, A, C, b, f,
dims[0], dims[1], YrA=YrA_GT, img=cn_filter)
#%%
cm.utils.visualization.view_patches_bar(Yr, A_, C_, b_, f_,
dims[0], dims[1], YrA=YrA_, img=cn_filter)
#%%
mapIdx = get_mapping(C_, C, A).astype(int)
if True:
corC = np.array([np.corrcoef(C_[mapIdx[n]], C[n])[0, 1] for n in range(N)])
corA = np.array([np.corrcoef(A_[:, mapIdx[n]].toarray().squeeze(), A[:, n])[0, 1]
for n in range(N)])
corC_cnmfe = np.array([np.corrcoef(C_cnmfe[n], C[n])[0, 1]
Npeaks = 5
traces = cnm.C + cnm.YrA
# TODO: todocument
idx_components, idx_components_bad = cm.components_evaluation.estimate_components_quality(
traces, Yr, cnm.A, cnm.C, cnm.b, cnm.f, final_frate=final_frate, Npeaks=Npeaks,
r_values_min=r_values_min, fitness_min=fitness_min, fitness_delta_min=fitness_delta_min, dview=dview)
print(('Keeping ' + str(len(idx_components)) +
' and discarding ' + str(len(idx_components_bad))))
#%%
A_, C_, YrA_, b_, f_ = (cnm.A[:, idx_components], cnm.C[idx_components],
cnm.YrA[idx_components], cnm.b, cnm.f)
#%%
YrA_GT = compute_residuals(np.array(Yr) - b0, A, b, C, f, dview=None)
cm.utils.visualization.view_patches_bar(Yr, A, C, b, f,
dims[0], dims[1], YrA=YrA_GT, img=cn_filter)
#%%
cm.utils.visualization.view_patches_bar(Yr, A_, C_, b_, f_,
dims[0], dims[1], YrA=YrA_, img=cn_filter)
#%%
mapIdx = get_mapping(C_, C, A).astype(int)
if True:
corC = np.array([np.corrcoef(C_[mapIdx[n]], C[n])[0, 1] for n in range(N)])
corA = np.array([np.corrcoef(A_[:, mapIdx[n]].toarray().squeeze(), A[:, n])[0, 1]
for n in range(N)])
corC_cnmfe = np.array([np.corrcoef(C_cnmfe[n], C[n])[0, 1] for n in range(N)])
low_rank_background=False,
update_background_components=False,
min_corr=min_corr,
min_pnr=min_pnr,
normalize_init=False,
deconvolve_options_init=None,
ring_size_factor=1.5,
center_psf=True)
cnm.fit(Y)
cnm.compute_residuals(Yr)
#%%
cnm.YrA = compute_residuals(np.array(Yr),
cnm.A,
cnm.b,
cnm.C,
cnm.f,
dview=dview)
#%%
if patches:
# %% DISCARD LOW QUALITY COMPONENT
final_frate = 10
r_values_min = 0.9 # threshold on space consistency
fitness_min = -1000 # threshold on time variability
# threshold on time variability (if nonsparse activity)
fitness_delta_min = -1000
Npeaks = 5
traces = cnm.C + cnm.YrA
# TODO: todocument
idx_components, idx_components_bad = cm.components_evaluation.estimate_components_quality(
traces,
def extract_masks(scan, mmap_scan, num_components=200, num_background_components=1,
merge_threshold=0.8, init_on_patches=True, init_method='greedy_roi',
soma_diameter=(14, 14), snmf_alpha=None, patch_size=(50, 50),
proportion_patch_overlap=0.2, num_components_per_patch=5,
num_processes=8, num_pixels_per_process=5000, fps=15,
p=0, ssub=2, tsub=2):
""" Extract masks from multi-photon scans using CNMF.
Uses constrained non-negative matrix factorization to find spatial components (masks)
and their fluorescence traces in a scan. Default values work well for somatic scans.
Performed operations are:
[Initialization on full image | Initialization on patches -> merge components] ->
spatial update -> temporal update -> merge components -> spatial update ->
temporal update
:param np.array scan: 3-dimensional scan (image_height, image_width, num_frames).
:param np.memmap mmap_scan: 2-d scan (image_height * image_width, num_frames)
:param int num_components: An estimate of the number of spatial components in the scan
:param int num_background_components: Number of components to model the background.
:param int merge_threshold: Maximal temporal correlation allowed between the activity
of overlapping components before merging them.
:param bool init_on_patches: If True, run the initialization methods on small patches
of the scan rather than on the whole image.
:param string init_method: Initialization method for the components.
'greedy_roi': Look for a gaussian-shaped patch, apply rank-1 NMF, store
components, calculate residual scan and repeat for num_components.
'sparse_nmf': Regularized non-negative matrix factorization (as impl. in sklearn)
:param (float, float) soma_diameter: Estimated neuron size in y and x (pixels). Used
in'greedy_roi' initialization to search for neurons of this size.
:param int snmf_alpha: Regularization parameter (alpha) for sparse NMF (if used).
:param (float, float) patch_size: Size of the patches in y and x (pixels).
:param float proportion_patch_overlap: Patches are sampled in a sliding window. This
controls how much overlap is between adjacent patches (0 for none, 0.9 for 90%).
:param int num_components_per_patch: Number of components per patch (used if
init_on_patches=True)
:param int num_processes: Number of processes to run in parallel. None for as many
processes as available cores.
:param int num_pixels_per_process: Number of pixels that a process handles each
iteration.
:param fps: Frame rate. Used for temporal downsampling and to remove bad components.
:returns: Weighted masks (image_height x image_width x num_components). Inferred
location of each component.
:returns: Denoised fluorescence traces (num_components x num_frames).
:returns: Masks for background components (image_height x image_width x
num_background_components).
:returns: Traces for background components (image_height x image_width x
num_background_components).
:returns: Raw fluorescence traces (num_components x num_frames). Fluorescence of each
component in the scan minus activity from other components and background.
..warning:: The produced number of components is not exactly what you ask for because
some components will be merged or deleted.
..warning:: Better results if scans are nonnegative.
"""
# Get some params
image_height, image_width, num_frames = scan.shape
# Start processes
log('Starting {} processes...'.format(num_processes))
pool = mp.Pool(processes=num_processes)
# Initialize components
log('Initializing components...')
if init_on_patches:
# TODO: Redo this (per-patch initialization) in a nicer/more efficient way
# Make sure they are integers
patch_size = np.array(patch_size)
half_patch_size = np.int32(np.round(patch_size / 2))
num_components_per_patch = int(round(num_components_per_patch))
patch_overlap = np.int32(np.round(patch_size * proportion_patch_overlap))
# Create options dictionary (needed for run_CNMF_patches)
options = {'patch': {'only_init_patch': True,
'rf': half_patch_size, 'stride': patch_overlap,
'remove_very_bad_comps': False},
'preprocess': {'check_nan': False},
'temporal': {'p': p},
'init': {'k': num_components_per_patch, 'gSig': np.array(soma_diameter)/2,
'gSiz': None, 'method_init': init_method, 'alpha_snmf': snmf_alpha,
'gnb': num_background_components, 'ssub': ssub, 'tsub': tsub,
'normalize_init': True,
'rolling_sum': True, 'rolling_length': 100}}
options_flat = {}
for option_key in options:
options_flat.update(options[option_key])
options_object = params.CNMFParams(**options_flat)
# Initialize per patch
res = map_reduce.run_CNMF_patches(mmap_scan.filename, (image_height, image_width, num_frames),
options_object, gnb=num_background_components, dview=pool)
initial_A, initial_C, YrA, initial_b, initial_f, pixels_noise, _ = res
# Merge spatially overlapping components
merged_masks = ['dummy']
while len(merged_masks) > 0:
R = utilities.compute_residuals(mmap_scan, initial_A, initial_b, initial_C, initial_f)
res = merging.merge_components(mmap_scan, initial_A, initial_b, initial_C, R,
initial_f, initial_C, sn_pix=pixels_noise,
temporal_params = {'p': 0, 'method': 'cvxpy'}, spatial_params='UNUSED',
dview=pool, thr=merge_threshold, mx=np.Inf)
initial_A, initial_C, num_components, merged_masks, S, bl, c1, neurons_noise, g , _, R = res
# Delete log files (one per patch)
log_files = glob.glob('caiman*_LOG_*')
for log_file in log_files:
os.remove(log_file)
else:
from scipy.sparse import csr_matrix
if init_method == 'greedy_roi':
res = _greedyROI(scan, num_components, soma_diameter, num_background_components)
log('Refining initial components (HALS)...')
res = initialization.hals(scan, res[0].reshape([image_height * image_width, -1], order='F'),
res[1], res[2].reshape([image_height * image_width, -1], order='F'),
res[3], maxIter=3)
initial_A, initial_C, initial_b, initial_f = res
else:
print('Warning: Running sparse_nmf initialization on the entire field of view '
'takes a lot of time.')
res = initialization.initialize_components(scan, K=num_components, nb=num_background_components,
method=init_method, alpha_snmf=snmf_alpha)
initial_A, initial_C, initial_b, initial_f, _ = res
initial_A = csr_matrix(initial_A)
log(initial_A.shape[-1], 'components found...')
# Remove bad components (based on spatial consistency and spiking activity)
log('Removing bad components...')
good_indices, _ = components_evaluation.estimate_components_quality(initial_C, scan,
initial_A, initial_C, initial_b, initial_f, final_frate=fps, r_values_min=0.7,
fitness_min=-20, fitness_delta_min=-20, dview=pool)
initial_A = initial_A[:, good_indices]
initial_C = initial_C[good_indices]
log(initial_A.shape[-1], 'components remaining...')
# Estimate noise per pixel
log('Calculating noise per pixel...')
pixels_noise, _ = pre_processing.get_noise_fft_parallel(mmap_scan, num_pixels_per_process, pool)
# Update masks
log('Updating masks...')
A, b, C, f = spatial.update_spatial_components(mmap_scan, initial_C, initial_f, initial_A, b_in=initial_b,
sn=pixels_noise, dims=(image_height, image_width),
method_exp='dilate', dview=pool,
n_pixels_per_process=num_pixels_per_process,
nb=num_background_components)
# Update traces (no impulse response modelling p=0)
log('Updating traces...')
res = temporal.update_temporal_components(mmap_scan, A, b, C, f, nb=num_background_components,
block_size=10000, p=0, method='cvxpy', dview=pool)
C, A, b, f, S, bl, c1, neurons_noise, g, YrA, _ = res
# Merge components
log('Merging overlapping (and temporally correlated) masks...')
merged_masks = ['dummy']
while len(merged_masks) > 0:
R = utilities.compute_residuals(mmap_scan, A, b, C, f)
res = merging.merge_components(mmap_scan, A, b, C, R, f, S, sn_pix=pixels_noise,
temporal_params={'p': 0, 'method': 'cvxpy'},
spatial_params='UNUSED', dview=pool, thr=merge_threshold, bl=bl, c1=c1,
sn=neurons_noise, g=g)
A, C, num_components, merged_masks, S, bl, c1, neurons_noise, g, _, R = res
# Refine masks
log('Refining masks...')
A, b, C, f = spatial.update_spatial_components(mmap_scan, C, f, A, b_in=b, sn=pixels_noise,
dims=(image_height, image_width),
method_exp='dilate', dview=pool,
n_pixels_per_process=num_pixels_per_process,
nb=num_background_components)
# Refine traces
log('Refining traces...')
res = temporal.update_temporal_components(mmap_scan, A, b, C, f, nb=num_background_components,
block_size=10000, p=0, method='cvxpy', dview=pool)
C, A, b, f, S, bl, c1, neurons_noise, g, YrA, _ = res
# Removing bad components (more stringent criteria)
log('Removing bad components...')
good_indices, _ = components_evaluation.estimate_components_quality(C + YrA, scan, A,
C, b, f, final_frate=fps, r_values_min=0.8, fitness_min=-40, fitness_delta_min=-40,
dview=pool)
A = A.toarray()[:, good_indices]
C = C[good_indices]
YrA = YrA[good_indices]
log(A.shape[-1], 'components remaining...')
# Stop processes
log('Done.')
pool.close()
# Get results
masks = A.reshape((image_height, image_width, -1), order='F') # h x w x num_components
traces = C # num_components x num_frames
background_masks = b.reshape((image_height, image_width, -1), order='F') # h x w x num_components
background_traces = f # num_background_components x num_frames
raw_traces = C + YrA # num_components x num_frames
# Rescale traces to match scan range
scaling_factor = np.sum(masks**2, axis=(0, 1)) / np.sum(masks, axis=(0, 1))
traces = traces * np.expand_dims(scaling_factor, -1)
raw_traces = raw_traces * np.expand_dims(scaling_factor, -1)
masks = masks / scaling_factor
background_scaling_factor = np.sum(background_masks**2, axis=(0, 1)) / np.sum(background_masks,
axis=(0,1))
background_traces = background_traces * np.expand_dims(background_scaling_factor, -1)
background_masks = background_masks / background_scaling_factor
return masks, traces, background_masks, background_traces, raw_traces