Example #1
0
def inspect_results(images, cnm, is_3d=False):
    """Inspect results by plotting contours of identified components
    against correlation image. The results of the algorithm are stored in
    the object cnm.estimates."""
    # Plot contours of found components
    if is_3d:
        cn = cm.local_correlations(images, swap_dim=False)
        cn[np.isnan(cn)] = 0
        return cnm, cn
    cn = cm.local_correlations(images.transpose(1, 2, 0))
    cn[np.isnan(cn)] = 0
    cnm.estimates.plot_contours_nb(img=cn)
    return cnm, cn
Example #2
0
 def view_corr(self):
     self.corr_images = []
     fig, axes = plt.subplots(1, self.planes, constrained_layout=True)
     for i,ax in enumerate(axes):
         image = self.motion_corrected_images[i]
         corr_im = cm.local_correlations(image, swap_dim=False)
         self.corr_images.append(corr_im)
         ax.imshow(image)
         ax.set_aspect('equal', 'box')
         ax.axis('off')
         ax.set_title(f'Plane {i}')
     fig.suptitle('Correlation Image')
Example #3
0
    def output_cnmf(self):  # , batch_dir, UUID, viewer_ref):
        pass
        filename = self.batch_dir + '/' + str(self.UUID)

        self.get_cnmf_results()

        Yr = pickle.load(open(filename + '_Yr.pikl', 'rb'))

        cnmb = pickle.load(open(filename + '_cnm-b.pikl', 'rb'))
        cnm_f = pickle.load(open(filename + '_cnm-f.pikl', 'rb'))
        cnmYrA = pickle.load(open(filename + '_cnm-YrA.pikl', 'rb'))

        img = tifffile.imread(self.batch_dir + '/' + str(self.UUID) + '.tiff')
        Cn = cm.local_correlations(img.transpose(1, 2, 0))
        Cn[np.isnan(Cn)] = 0

        self.visualization = cm.utils.visualization.view_patches_bar(Yr, self.cnmA[:, self.idx_components],
                                                                     self.cnmC[self.idx_components], cnmb, cnm_f,
                                                                     self.dims[0], self.dims[1],
                                                                     YrA=cnmYrA[self.idx_components], img=Cn)
Example #4
0
t2 = time.time() - t1

#%% LOAD MEMMAP FILE
# fname_new='Yr_d1_501_d2_398_d3_1_order_F_frames_369_.mmap'
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
Y = np.reshape(Yr, dims + (T,), order='F')
m_images = cm.movie(images)
#%%  checks on movies (might take time if large!)
# if np.min(images) < 0:
#    raise Exception('Movie too negative, add_to_movie should be larger')
# if np.sum(np.isnan(images)) > 0:
#    raise Exception('Movie contains nan! You did not remove enough borders')
#%% correlation image
Cn = cm.local_correlations(Y)
Cn[np.isnan(Cn)] = 0
pl.imshow(Cn, cmap='gray', vmax=.55)
#%% some parameter settings
# order of the autoregressive fit to calcium imaging in general one (slow gcamps) or two (fast gcamps fast scanning)
p = params_movie['p']
# merging threshold, max correlation allowed
merge_thresh = params_movie['merge_thresh']
# half-size of the patches in pixels. rf=25, patches are 50x50
rf = params_movie['rf']
# amounpl.it of overlap between the patches in pixels
stride_cnmf = params_movie['stride_cnmf']
# number of components per patch
K = params_movie['K']
# if dendritic. In this case you need to set init_method to sparse_nmf
is_dendrites = params_movie['is_dendrites']
Example #5
0
def gen_data(dims=(48, 48), N=10, sig=(3, 3), tau=1., noise=.3, T=2000,
             framerate=30, firerate=.5, seed=3, cmap=False, truncate=np.exp(-2),
             difference_of_Gaussians=True, fluctuating_bkgrd=[50, 300]):
    bkgrd = 10  # fluorescence baseline
    np.random.seed(seed)
    boundary = 4
    M = int(N * 1.5)
    # centers = boundary + (np.array(GeneralizedHalton(2, seed).get(M)) *
    #                       (np.array(dims) - 2 * boundary)).astype('uint16')
    centers = boundary + (np.random.rand(M, 2) *
                          (np.array(dims) - 2 * boundary)).astype('uint16')
    trueA = np.zeros(dims + (M,), dtype='float32')
    for i in range(M):
        trueA[tuple(centers[i]) + (i,)] = 1.
    if difference_of_Gaussians:
        q = .75
        for n in range(M):
            s = (.67 + .33 * np.random.rand(2)) * np.array(sig)
            tmp = gaussian_filter(trueA[:, :, n], s)
            trueA[:, :, n] = np.maximum(tmp - gaussian_filter(trueA[:, :, n], q * s) *
                                        q**2 * (.2 + .6 * np.random.rand()), 0)

    else:
        for n in range(M):
            s = [ss * (.75 + .25 * np.random.rand()) for ss in sig]
            trueA[:, :, n] = gaussian_filter(trueA[:, :, n], s)
    trueA = trueA.reshape((-1, M), order='F')
    trueA *= (trueA >= trueA.max(0) * truncate)
    trueA /= np.linalg.norm(trueA, 2, 0)
    keep = np.ones(M, dtype=bool)
    overlap = trueA.T.dot(trueA) - np.eye(M)
    while keep.sum() > N:
        keep[np.argmax(overlap * np.outer(keep, keep)) % M] = False
    trueA = trueA[:, keep]
    trueS = np.random.rand(N, T) < firerate / float(framerate)
    trueS[:, 0] = 0
    for i in range(N // 2):
        trueS[i, :500 + i * T // N * 2 // 3] = 0
    trueC = trueS.astype('float32')
    for i in range(N):
        # * (.9 + .2 * np.random.rand())))
        gamma = np.exp(-1. / (tau * framerate))
        for t in range(1, T):
            trueC[i, t] += gamma * trueC[i, t - 1]

    if fluctuating_bkgrd:
        K = np.array([[np.exp(-(i - j)**2 / 2. / fluctuating_bkgrd[0]**2)
                       for i in range(T)] for j in range(T)])
        ch = np.linalg.cholesky(K + 1e-10 * np.eye(T))
        truef = 1e-2 * ch.dot(np.random.randn(T)).astype('float32') / bkgrd
        truef -= truef.mean()
        truef += 1
        K = np.array([[np.exp(-(i - j)**2 / 2. / fluctuating_bkgrd[1]**2)
                       for i in range(dims[0])] for j in range(dims[0])])
        ch = np.linalg.cholesky(K + 1e-10 * np.eye(dims[0]))
        trueb = 3 * 1e-2 * \
            np.outer(
                *ch.dot(np.random.randn(dims[0], 2)).T).ravel().astype('float32')
        trueb -= trueb.mean()
        trueb += 1
    else:
        truef = np.ones(T, dtype='float32')
        trueb = np.ones(np.prod(dims), dtype='float32')
    trueb *= bkgrd
    Yr = np.outer(trueb, truef) + noise * np.random.randn(
        * (np.prod(dims), T)).astype('float32') + trueA.dot(trueC)

    if cmap:
        import matplotlib.pyplot as plt
        import caiman as cm
        Y = np.reshape(Yr, dims + (T,), order='F')
        Cn = cm.local_correlations(Y)
        plt.figure(figsize=(20, 3))
        plt.plot(trueC.T)
        plt.figure(figsize=(20, 3))
        plt.plot((trueA.T.dot(Yr - bkgrd) / np.sum(trueA**2, 0).reshape(-1, 1)).T)
        plt.figure(figsize=(12, 4))
        plt.subplot(131)
        plt.scatter(*centers[keep].T[::-1], c='g')
        plt.scatter(*centers[~keep].T[::-1], c='r')
        plt.imshow(Y[:T // 10 * 10].reshape(dims +
                                            (T // 10, 10)).mean(-1).max(-1), cmap=cmap)
        plt.title('Max')
        plt.subplot(132)
        plt.scatter(*centers[keep].T[::-1], c='g')
        plt.scatter(*centers[~keep].T[::-1], c='r')
        plt.imshow(Y.mean(-1), cmap=cmap)
        plt.title('Mean')
        plt.subplot(133)
        plt.scatter(*centers[keep].T[::-1], c='g')
        plt.scatter(*centers[~keep].T[::-1], c='r')
        plt.imshow(Cn, cmap=cmap)
        plt.title('Correlation')
        plt.show()
    return Yr, trueC, trueS, trueA, trueb, truef, centers, dims
Example #6
0
def initialize_movie_online(Y,
                            K,
                            gSig,
                            rf,
                            stride,
                            base_name,
                            p=1,
                            merge_thresh=0.95,
                            rval_thr_online=0.9,
                            thresh_fitness_delta_online=-30,
                            thresh_fitness_raw_online=-50,
                            rval_thr_init=.5,
                            thresh_fitness_delta_init=-20,
                            thresh_fitness_raw_init=-20,
                            rval_thr_refine=0.95,
                            thresh_fitness_delta_refine=-100,
                            thresh_fitness_raw_refine=-100,
                            final_frate=10,
                            Npeaks=10,
                            single_thread=True,
                            dview=None,
                            n_processes=None):
    """
    Initialize movie using CNMF on minibatch. See CNMF parameters
    """

    _, d1, d2 = Y.shape
    dims = (d1, d2)
    Yr = Y.to_2D().T
    # merging threshold, max correlation allowed
    # order of the autoregressive system
    #T = Y.shape[0]
    base_name = base_name + '.mmap'
    fname_new = Y.save(base_name, order='C')
    #%
    Yr, dims, T = cm.load_memmap(fname_new)
    d1, d2 = dims
    images = np.reshape(Yr.T, [T] + list(dims), order='F')
    Y = np.reshape(Yr, dims + (T, ), order='F')
    Cn2 = cm.local_correlations(Y)
    #    pl.imshow(Cn2)
    #%
    #% RUN ALGORITHM ON PATCHES
    #    pl.close('all')
    cnm_init = cm.source_extraction.cnmf.CNMF(
        n_processes,
        method_init='greedy_roi',
        k=K,
        gSig=gSig,
        merge_thresh=merge_thresh,
        p=0,
        dview=dview,
        Ain=None,
        rf=rf,
        stride=stride,
        method_deconvolution='oasis',
        skip_refinement=False,
        normalize_init=False,
        options_local_NMF=None,
        minibatch_shape=100,
        minibatch_suff_stat=5,
        update_num_comps=True,
        rval_thr=rval_thr_online,
        thresh_fitness_delta=thresh_fitness_delta_online,
        thresh_fitness_raw=thresh_fitness_raw_online,
        batch_update_suff_stat=True,
        max_comp_update_shape=5)

    cnm_init = cnm_init.fit(images)
    A_tot = cnm_init.A
    C_tot = cnm_init.C
    YrA_tot = cnm_init.YrA
    b_tot = cnm_init.b
    f_tot = cnm_init.f

    print(('Number of components:' + str(A_tot.shape[-1])))

    #%

    traces = C_tot + YrA_tot
    #        traces_a=traces-scipy.ndimage.percentile_filter(traces,8,size=[1,np.shape(traces)[-1]/5])
    #        traces_b=np.diff(traces,axis=1)
    fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = cm.components_evaluation.evaluate_components(
        Y,
        traces,
        A_tot,
        C_tot,
        b_tot,
        f_tot,
        final_frate,
        remove_baseline=True,
        N=5,
        robust_std=False,
        Athresh=0.1,
        Npeaks=Npeaks,
        thresh_C=0.3)

    idx_components_r = np.where(r_values >= rval_thr_init)[0]
    idx_components_raw = np.where(fitness_raw < thresh_fitness_raw_init)[0]
    idx_components_delta = np.where(
        fitness_delta < thresh_fitness_delta_init)[0]

    idx_components = np.union1d(idx_components_r, idx_components_raw)
    idx_components = np.union1d(idx_components, idx_components_delta)
    idx_components_bad = np.setdiff1d(list(range(len(traces))), idx_components)

    print(('Keeping ' + str(len(idx_components)) + ' and discarding  ' +
           str(len(idx_components_bad))))

    A_tot = A_tot.tocsc()[:, idx_components]
    C_tot = C_tot[idx_components]
    #%
    cnm_refine = cm.source_extraction.cnmf.CNMF(
        n_processes,
        method_init='greedy_roi',
        k=A_tot.shape,
        gSig=gSig,
        merge_thresh=merge_thresh,
        rf=None,
        stride=None,
        p=p,
        dview=dview,
        Ain=A_tot,
        Cin=C_tot,
        f_in=f_tot,
        method_deconvolution='oasis',
        skip_refinement=True,
        normalize_init=False,
        options_local_NMF=None,
        minibatch_shape=100,
        minibatch_suff_stat=5,
        update_num_comps=True,
        rval_thr=rval_thr_refine,
        thresh_fitness_delta=thresh_fitness_delta_refine,
        thresh_fitness_raw=thresh_fitness_raw_refine,
        batch_update_suff_stat=True,
        max_comp_update_shape=5)

    cnm_refine = cnm_refine.fit(images)
    #%
    A, C, b, f, YrA, sn = cnm_refine.A, cnm_refine.C, cnm_refine.b, cnm_refine.f, cnm_refine.YrA, cnm_refine.sn
    #%
    final_frate = 10
    Npeaks = 10
    traces = C + YrA


    fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = \
        cm.components_evaluation.evaluate_components(Y, traces, A, C, b, f, final_frate, remove_baseline=True,
                                          N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks,  thresh_C=0.3)

    idx_components_r = np.where(r_values >= rval_thr_refine)[0]
    idx_components_raw = np.where(fitness_raw < thresh_fitness_raw_refine)[0]
    idx_components_delta = np.where(
        fitness_delta < thresh_fitness_delta_refine)[0]

    idx_components = np.union1d(idx_components_r, idx_components_raw)
    idx_components = np.union1d(idx_components, idx_components_delta)
    idx_components_bad = np.setdiff1d(list(range(len(traces))), idx_components)

    print(' ***** ')
    print((len(traces)))
    print((len(idx_components)))
    #%
    cnm_refine.idx_components = idx_components
    cnm_refine.idx_components_bad = idx_components_bad
    cnm_refine.r_values = r_values
    cnm_refine.fitness_raw = fitness_raw
    cnm_refine.fitness_delta = fitness_delta
    cnm_refine.Cn2 = Cn2

    #%

    #    cnm_init.dview = None
    #    save_object(cnm_init,fls[0][:-4]+ '_DS_' + str(ds)+ '_init.pkl')

    return cnm_refine, Cn2, fname_new
Example #7
0
else:
    print('One file only, not saving!')
    fname_new = name_new[0]
#%%
# fname_new='Yr_d1_501_d2_398_d3_1_order_F_frames_369_.mmap'
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
Y = np.reshape(Yr, dims + (T, ), order='F')
#%%
if np.min(images) < 0:
    raise Exception('Movie too negative, add_to_movie should be larger')
if np.sum(np.isnan(images)) > 0:
    raise Exception('Movie contains nan! You did not remove enough borders')
#%%
Cn = cm.local_correlations(Y[:, :, :3000])
pl.imshow(Cn, cmap='gray')

#%%
if not is_patches:
    K = 35  # number of neurons expected per patch
    gSig = [7, 7]  # expected half size of neurons
    merge_thresh = 0.8  # merging threshold, max correlation allowed
    p = 2  # order of the autoregressive system
    cnm = cnmf.CNMF(n_processes,
                    method_init=init_method,
                    k=K,
                    gSig=gSig,
                    merge_thresh=merge_thresh,
                    p=p,
                    dview=dview,
Example #8
0
name_new.sort()
print(name_new)
#%%
fname_new = cm.save_memmap_join(
    name_new, base_name='Yr', n_chunks=12, dview=dview)
#%%
# fname_new='Yr_d1_501_d2_398_d3_1_order_F_frames_369_.mmap'
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
Y = np.reshape(Yr, dims + (T,), order='F')
#%%
if np.min(images) < 0:
    raise Exception('Movie too negative, add_to_movie should be larger')
#%%
Cn = cm.local_correlations(Y[:, :, :3000])
pl.imshow(Cn, cmap='gray')

#%%
# THIS METHOd CAN GIVE POSSIBLY INCONSISTENT RESULTS ON SOMAS WHEN NOT USED WITH PATCHES
init_method = 'sparse_nmf'
alpha_snmf = 10e3  # this controls sparsity!

#%%
rf = 30  # half-size of the patches in pixels. rf=25, patches are 50x50
stride = 10  # amounpl.it of overlap between the patches in pixels
K = 10  # number of neurons expected per patch
gSig = [6, 6]  # expected half size of neurons
merge_thresh = 0.8  # merging threshold, max correlation allowed
p = 1  # order of the autoregressive system
memory_fact = 1  # unitless number accounting how much memory should be used. You will need to try different values to see which one would work the default is OK for a 16 GB system
Example #9
0
def main():
    pass  # For compatibility between running under Spyder and the CLI

    #%% First setup some parameters

    # dataset dependent parameters
    display_images = False  # Set this to true to show movies and plots
    fname = ['Sue_2x_3000_40_-46.tif']  # filename to be processed
    fr = 30  # imaging rate in frames per second
    decay_time = 0.4  # length of a typical transient in seconds

    # motion correction parameters
    niter_rig = 1  # number of iterations for rigid motion correction
    max_shifts = (6, 6)  # maximum allow rigid shift
    # for parallelization split the movies in  num_splits chuncks across time
    splits_rig = 56
    # start a new patch for pw-rigid motion correction every x pixels
    strides = (48, 48)
    # overlap between pathes (size of patch strides+overlaps)
    overlaps = (24, 24)
    # for parallelization split the movies in  num_splits chuncks across time
    splits_els = 56
    upsample_factor_grid = 4  # upsample factor to avoid smearing when merging patches
    # maximum deviation allowed for patch with respect to rigid shifts
    max_deviation_rigid = 3

    # parameters for source extraction and deconvolution
    p = 1  # order of the autoregressive system
    gnb = 2  # number of global background components
    merge_thresh = 0.8  # merging threshold, max correlation allowed
    # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
    rf = 15
    stride_cnmf = 6  # amount of overlap between the patches in pixels
    K = 4  # number of components per patch
    gSig = [4, 4]  # expected half size of neurons
    # initialization method (if analyzing dendritic data using 'sparse_nmf')
    init_method = 'greedy_roi'
    is_dendrites = False  # flag for analyzing dendritic data
    # sparsity penalty for dendritic data analysis through sparse NMF
    alpha_snmf = None

    # parameters for component evaluation
    min_SNR = 2.5  # signal to noise ratio for accepting a component
    rval_thr = 0.8  # space correlation threshold for accepting a component
    cnn_thr = 0.8  # threshold for CNN based classifier

    #%% download the dataset if it's not present in your folder
    if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
        fname = [download_demo(fname[0])]

#%% play the movie
# playing the movie using opencv. It requires loading the movie in memory. To
# close the video press q

    m_orig = cm.load_movie_chain(fname[:1])
    downsample_ratio = 0.2
    offset_mov = -np.min(m_orig[:100])
    moviehandle = m_orig.resize(1, 1, downsample_ratio)
    if display_images:
        moviehandle.play(gain=10, offset=offset_mov, fr=30, magnification=2)

#%% start a cluster for parallel processing
    c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
                                                     n_processes=None,
                                                     single_thread=False)

    #%%% MOTION CORRECTION
    # first we create a motion correction object with the parameters specified
    min_mov = cm.load(fname[0], subindices=range(200)).min()
    # this will be subtracted from the movie to make it non-negative

    mc = MotionCorrect(fname[0],
                       min_mov,
                       dview=dview,
                       max_shifts=max_shifts,
                       niter_rig=niter_rig,
                       splits_rig=splits_rig,
                       strides=strides,
                       overlaps=overlaps,
                       splits_els=splits_els,
                       upsample_factor_grid=upsample_factor_grid,
                       max_deviation_rigid=max_deviation_rigid,
                       shifts_opencv=True,
                       nonneg_movie=True)
    # note that the file is not loaded in memory

    #%% Run piecewise-rigid motion correction using NoRMCorre
    mc.motion_correct_pwrigid(save_movie=True)
    m_els = cm.load(mc.fname_tot_els)
    bord_px_els = np.ceil(
        np.maximum(np.max(np.abs(mc.x_shifts_els)),
                   np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
    # maximum shift to be used for trimming against NaNs
    #%% compare with original movie
    moviehandle = cm.concatenate([
        m_orig.resize(1, 1, downsample_ratio) + offset_mov,
        m_els.resize(1, 1, downsample_ratio)
    ],
                                 axis=2)
    display_images = False
    if display_images:
        moviehandle.play(fr=60, q_max=99.5, magnification=2,
                         offset=0)  # press q to exit

#%% MEMORY MAPPING
# memory map the file in order 'C'
    fnames = mc.fname_tot_els  # name of the pw-rigidly corrected file.
    border_to_0 = bord_px_els  # number of pixels to exclude
    fname_new = cm.save_memmap(fnames,
                               base_name='memmap_',
                               order='C',
                               border_to_0=bord_px_els)  # exclude borders

    # now load the file
    Yr, dims, T = cm.load_memmap(fname_new)
    d1, d2 = dims
    images = np.reshape(Yr.T, [T] + list(dims), order='F')
    # load frames in python format (T x X x Y)

    #%% restart cluster to clean up memory
    cm.stop_server(dview=dview)
    c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
                                                     n_processes=None,
                                                     single_thread=False)

    #%% RUN CNMF ON PATCHES

    # First extract spatial and temporal components on patches and combine them
    # for this step deconvolution is turned off (p=0)
    t1 = time.time()

    cnm = cnmf.CNMF(n_processes=1,
                    k=K,
                    gSig=gSig,
                    merge_thresh=merge_thresh,
                    p=0,
                    dview=dview,
                    rf=rf,
                    stride=stride_cnmf,
                    memory_fact=1,
                    method_init=init_method,
                    alpha_snmf=alpha_snmf,
                    only_init_patch=False,
                    gnb=gnb,
                    border_pix=bord_px_els)
    cnm = cnm.fit(images)

    #%% plot contours of found components
    Cn = cm.local_correlations(images.transpose(1, 2, 0))
    Cn[np.isnan(Cn)] = 0
    plt.figure()
    crd = plot_contours(cnm.A, Cn, thr=0.9)
    plt.title('Contour plots of found components')

    #%% COMPONENT EVALUATION
    # the components are evaluated in three ways:
    #   a) the shape of each component must be correlated with the data
    #   b) a minimum peak SNR is required over the length of a transient
    #   c) each shape passes a CNN based classifier

    idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
        estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
                                         cnm.YrA, fr, decay_time, gSig, dims,
                                         dview=dview, min_SNR=min_SNR,
                                         r_values_min=rval_thr, use_cnn=False,
                                         thresh_cnn_min=cnn_thr)

    #%% PLOT COMPONENTS

    if display_images:
        plt.figure()
        plt.subplot(121)
        crd_good = cm.utils.visualization.plot_contours(cnm.A[:,
                                                              idx_components],
                                                        Cn,
                                                        thr=.8,
                                                        vmax=0.75)
        plt.title('Contour plots of accepted components')
        plt.subplot(122)
        crd_bad = cm.utils.visualization.plot_contours(
            cnm.A[:, idx_components_bad], Cn, thr=.8, vmax=0.75)
        plt.title('Contour plots of rejected components')

#%% VIEW TRACES (accepted and rejected)

    if display_images:
        view_patches_bar(Yr,
                         cnm.A.tocsc()[:, idx_components],
                         cnm.C[idx_components],
                         cnm.b,
                         cnm.f,
                         dims[0],
                         dims[1],
                         YrA=cnm.YrA[idx_components],
                         img=Cn)

        view_patches_bar(Yr,
                         cnm.A.tocsc()[:, idx_components_bad],
                         cnm.C[idx_components_bad],
                         cnm.b,
                         cnm.f,
                         dims[0],
                         dims[1],
                         YrA=cnm.YrA[idx_components_bad],
                         img=Cn)

#%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
    A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[
        idx_components], cnm.b, cnm.f
    cnm2 = cnmf.CNMF(n_processes=1,
                     k=A_in.shape[-1],
                     gSig=gSig,
                     p=p,
                     dview=dview,
                     merge_thresh=merge_thresh,
                     Ain=A_in,
                     Cin=C_in,
                     b_in=b_in,
                     f_in=f_in,
                     rf=None,
                     stride=None,
                     gnb=gnb,
                     method_deconvolution='oasis',
                     check_nan=True)

    cnm2 = cnm2.fit(images)

    #%% Extract DF/F values

    F_dff = detrend_df_f(cnm2.A,
                         cnm2.b,
                         cnm2.C,
                         cnm2.f,
                         YrA=cnm2.YrA,
                         quantileMin=8,
                         frames_window=250)

    #%% Show final traces
    cnm2.view_patches(Yr, dims=dims, img=Cn)

    #%% STOP CLUSTER and clean up log files
    cm.stop_server(dview=dview)
    log_files = glob.glob('*_LOG_*')
    for log_file in log_files:
        os.remove(log_file)

#%% reconstruct denoised movie
    denoised = cm.movie(cnm2.A.dot(cnm2.C) + cnm2.b.dot(cnm2.f)).reshape(
        dims + (-1, ), order='F').transpose([2, 0, 1])

    #%% play along side original data
    moviehandle = cm.concatenate([
        m_els.resize(1, 1, downsample_ratio),
        denoised.resize(1, 1, downsample_ratio)
    ],
                                 axis=2)
    if display_images:
        moviehandle.play(fr=60, gain=15, magnification=2,
                         offset=0)  # press q to exit
Example #10
0
                gSig=gSig,
                merge_thresh=merge_thresh,
                p=0,
                dview=dview,
                rf=rf,
                stride=stride_cnmf,
                memory_fact=1,
                method_init=init_method,
                alpha_snmf=alpha_snmf,
                only_init_patch=False,
                gnb=gnb,
                border_pix=bord_px_els)
cnm = cnm.fit(images)

#%% plot contours of found components
Cn = cm.local_correlations(images.transpose(1, 2, 0))
Cn[np.isnan(Cn)] = 0
plt.figure()
crd = plot_contours(cnm.A, Cn, thr=0.9)
plt.title('Contour plots of found components')

#%% COMPONENT EVALUATION
# the components are evaluated in three ways:
#   a) the shape of each component must be correlated with the data
#   b) a minimum peak SNR is required over the length of a transient
#   c) each shape passes a CNN based classifier

idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
    estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
                                     cnm.YrA, fr, decay_time, gSig, dims,
                                     dview = dview, min_SNR=min_SNR,
Example #11
0
def process_data(haussio_data,
                 mask=None,
                 p=2,
                 nrois_init=400,
                 roi_iceberg=0.9):
    if mask is not None:
        raise RuntimeError("mask not supported in cnmf.process_data")

    fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat'
    shapefn = os.path.join(haussio_data.dirname_comp,
                           haussio.THOR_RAW_FN[:-3] + "shape.npy")
    shape = np.load(shapefn)
    if len(shape) == 5:
        d1, d2 = shape[2], shape[3]
        fn_mmap = get_mmap_name('Yr', shape[2], shape[3], shape[0])
    else:
        d1, d2 = shape[1], shape[2]
        fn_mmap = get_mmap_name('Yr', shape[1], shape[2], shape[0])
    fn_mmap = os.path.join(haussio_data.dirname_comp, fn_mmap)
    print(fn_mmap, os.path.exists(fn_mmap), d1, d2)

    if not os.path.exists(fn_cnmf):
        # fn_raw = os.path.join(haussio_data.dirname_comp, haussio.THOR_RAW_FN)
        fn_sima = haussio_data.dirname_comp + '.sima'
        fnames = [
            fn_sima,
        ]
        fnames.sort()
        print(fnames)
        fnames = fnames

        final_frate = 1.0 / haussio_data.dt
        downsample_factor = 1  # use .2 or .1 if file is large and you want a quick answer
        final_frate *= downsample_factor

        c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
                                                         n_processes=None,
                                                         single_thread=False)

        idx_xy = None
        base_name = 'Yr'
        name_new = cm.save_memmap_each(fnames,
                                       dview=dview,
                                       base_name=base_name,
                                       resize_fact=(1, 1, downsample_factor),
                                       remove_init=0,
                                       idx_xy=idx_xy)
        name_new.sort()
        print(name_new)

        if len(name_new) > 1:
            fname_new = cm.save_memmap_join(name_new,
                                            base_name='Yr',
                                            n_chunks=12,
                                            dview=dview)
        else:
            sys.stdout.write('One file only, not saving\n')
            fname_new = name_new[0]

        print("fname_new: " + fname_new)

        Yr, dims, T = cm.load_memmap(fname_new)
        Y = np.reshape(Yr, dims + (T, ), order='F')
        Cn = cm.local_correlations(Y)

        K = nrois_init  # number of neurons expected per patch
        gSig = [15, 15]  # expected half size of neurons
        merge_thresh = 0.8  # merging threshold, max correlation allowed
        p = 2  #order of the autoregressive system
        options = caiman_cnmf.utilities.CNMFSetParms(Y,
                                                     NCPUS,
                                                     p=p,
                                                     gSig=gSig,
                                                     K=K,
                                                     ssub=2,
                                                     tsub=2)

        Yr, sn, g, psx = caiman_cnmf.pre_processing.preprocess_data(
            Yr, dview=dview, **options['preprocess_params'])
        Atmp, Ctmp, b_in, f_in, center = caiman_cnmf.initialization.initialize_components(
            Y, **options['init_params'])

        Ain, Cin = Atmp, Ctmp
        A, b, Cin, f_in = caiman_cnmf.spatial.update_spatial_components(
            Yr,
            Cin,
            f_in,
            Ain,
            sn=sn,
            dview=dview,
            **options['spatial_params'])

        options['temporal_params'][
            'p'] = 0  # set this to zero for fast updating without deconvolution
        C, A, b, f, S, bl, c1, neurons_sn, g, YrA = caiman_cnmf.temporal.update_temporal_components(
            Yr,
            A,
            b,
            Cin,
            f_in,
            bl=None,
            c1=None,
            sn=None,
            g=None,
            **options['temporal_params'])

        A_m, C_m, nr_m, merged_ROIs, S_m, bl_m, c1_m, sn_m, g_m = caiman_cnmf.merging.merge_components(
            Yr,
            A,
            b,
            C,
            f,
            S,
            sn,
            options['temporal_params'],
            options['spatial_params'],
            dview=dview,
            bl=bl,
            c1=c1,
            sn=neurons_sn,
            g=g,
            thr=merge_thresh,
            mx=50,
            fast_merge=True)

        A2, b2, C2, f = caiman_cnmf.spatial.update_spatial_components(
            Yr, C_m, f, A_m, sn=sn, dview=dview, **options['spatial_params'])
        options['temporal_params'][
            'p'] = p  # set it back to original value to perform full deconvolution
        C2, A2, b2, f2, S2, bl2, c12, neurons_sn2, g21, YrA = caiman_cnmf.temporal.update_temporal_components(
            Yr,
            A2,
            b2,
            C2,
            f,
            dview=dview,
            bl=None,
            c1=None,
            sn=None,
            g=None,
            **options['temporal_params'])

        tB = np.minimum(-2, np.floor(-5. / 30 * final_frate))
        tA = np.maximum(5, np.ceil(25. / 30 * final_frate))
        Npeaks = 10
        traces = C2 + YrA
        fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = \
            evaluate_components(
                Y, traces, A2, C2, b2, f2, final_frate, remove_baseline=True, N=5,
                robust_std=False, Athresh=0.1, Npeaks=Npeaks, thresh_C=0.3)

        idx_components_r = np.where(r_values >= .6)[0]
        idx_components_raw = np.where(fitness_raw < -60)[0]
        idx_components_delta = np.where(fitness_delta < -20)[0]

        min_radius = gSig[0] - 2
        masks_ws, idx_blobs, idx_non_blobs = extract_binary_masks_blob(
            A2.tocsc(),
            min_radius,
            dims,
            num_std_threshold=1,
            minCircularity=0.6,
            minInertiaRatio=0.2,
            minConvexity=.8)

        idx_components = np.union1d(idx_components_r, idx_components_raw)
        idx_components = np.union1d(idx_components, idx_components_delta)
        idx_blobs = np.intersect1d(idx_components, idx_blobs)
        idx_components_bad = np.setdiff1d(range(len(traces)), idx_components)

        A2 = A2.tocsc()[:, idx_components]
        C2 = C2[idx_components, :]
        YrA = YrA[idx_components, :]
        S2 = S2[idx_components, :]

        # A: spatial components (ROIs)
        # C: denoised [Ca2+]
        # YrA: residuals ("noise", i.e. traces = C+YrA)
        # S: Spikes
        savemat(fn_cnmf, {"A": A2, "C": C2, "YrA": YrA, "S": S2, "bl": bl2})

    else:
        resdict = loadmat(fn_cnmf)
        A2 = resdict["A"]
        C2 = resdict["C"]
        YrA = resdict["YrA"]
        S2 = resdict["S"]
        bl2 = resdict["bl"]
        Yr, dims, T = cm.load_memmap(fn_mmap)
        dims = dims[1:]
        Y = np.reshape(Yr, dims + (T, ), order='F')

    proj_fn = haussio_data.dirname_comp + "_proj.npy"
    if not os.path.exists(proj_fn):
        zproj = utils.zproject(np.transpose(Y, (2, 0, 1)))
        np.save(proj_fn, zproj)
    else:
        zproj = np.load(proj_fn)

    # DF_F, DF = cse.extract_DF_F(Y.reshape(d1*d2, T), A2, C2)

    # t0 = time.time()
    # sys.stdout.write("Ordering components... ")
    # sys.stdout.flush()
    # A_or, C_or, srt = cse.order_components(A2, C2)
    # sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))

    cm.stop_server()

    polygons = contour(A2, Y.shape[0], Y.shape[1], thr=roi_iceberg)
    rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons])

    return rois, C2, zproj, S2, Y, YrA
                               base_name=None)
#%%
n_chunks = 6  # increase this number if you have memory issues at this point
fname_new = cm.save_memmap_join(name_new,
                                base_name='MOV_EXAMPLE_20160706154257__',
                                n_chunks=6,
                                dview=dview)
#%%
Yr, dims, T = cm.load_memmap(
    'MOV_EXAMPLE_20160706154257___d1_150_d2_330_d3_1_order_C_frames_2300_.mmap'
)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
Y = np.reshape(Yr, dims + (T, ), order='F')
#%%
Cn = cm.local_correlations(Y[:, :, :3000], swap_dim=True)
pl.imshow(Cn, cmap='gray')
#%%
rf = 10  # half-size of the patches in pixels. rf=25, patches are 50x50
stride = 4  # amounpl.it of overlap between the patches in pixels
K = 4  # number of neurons expected per patch
gSig = [5, 5]  # expected half size of neurons
merge_thresh = 0.8  # merging threshold, max correlation allowed
p = 2  # order of the autoregressive system
memory_fact = 1  # unitless number accounting how much memory should be used. You will need to try different values to see which one would work the default is OK for a 16 GB system
save_results = False
#%% RUN ALGORITHM ON PATCHES
cnm = cnmf.CNMF(n_processes,
                k=K,
                gSig=gSig,
                merge_thresh=0.8,
Example #13
0
def main():
    pass # For compatibility between running under Spyder and the CLI

#%% First setup some parameters for data and motion correction

    # dataset dependent parameters
    fname = ['Sue_2x_3000_40_-46.tif']  # filename to be processed
    fr = 30                             # imaging rate in frames per second
    decay_time = 0.4                    # length of a typical transient in seconds
    dxy = (2., 2.)                      # spatial resolution in x and y in (um per pixel)
    max_shift_um = (12., 12.)           # maximum shift in um
    patch_motion_um = (100., 100.)      # patch size for non-rigid motion correction in um
    
    # motion correction parameters
    pwrigid_motion_correct = True       # flag to select rigid vs pw_rigid motion correction
    max_shifts = tuple([int(a/b) for a, b in zip(max_shift_um, dxy)])                 
                                        # maximum allow rigid shift in pixels
    # for parallelization split the movies in  num_splits chuncks across time
    splits_rig = 56
    # start a new patch for pw-rigid motion correction every x pixels
    strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
    # overlap between pathes (size of patch strides+overlaps)
    overlaps = (24, 24)
    # for parallelization split the movies in  num_splits chuncks across time
    splits_els = 56
    upsample_factor_grid = 4    # upsample factor to avoid smearing when merging patches
    # maximum deviation allowed for patch with respect to rigid shifts
    max_deviation_rigid = 3

#%% download the dataset if it's not present in your folder
    if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
        fname = [download_demo(fname[0])]

#%% play the movie
    # playing the movie using opencv. It requires loading the movie in memory.
    # To close the video press q
    display_images = False

    if display_images:
        m_orig = cm.load_movie_chain(fname)
        downsample_ratio = 0.2
        moviehandle = m_orig.resize(1, 1, downsample_ratio)
        moviehandle.play(q_max=99.5, fr=60, magnification=2)

#%% start a cluster for parallel processing
    c, dview, n_processes = cm.cluster.setup_cluster(
        backend='local', n_processes=None, single_thread=False)


#%%% MOTION CORRECTION
    # first we create a motion correction object with the parameters specified
    min_mov = cm.load(fname[0], subindices=range(200)).min()
    # this will be subtracted from the movie to make it non-negative

    mc = MotionCorrect(fname, min_mov, dview=dview, max_shifts=max_shifts, 
                       splits_rig=splits_rig,
                       strides=strides, overlaps=overlaps, 
                       splits_els=splits_els, border_nan='copy',
                       upsample_factor_grid=upsample_factor_grid,
                       max_deviation_rigid=max_deviation_rigid,
                       shifts_opencv=True, nonneg_movie=True)
    # note that the file is not loaded in memory

#%% Run piecewise-rigid motion correction using NoRMCorre
    if pwrigid_motion_correct:
        mc.motion_correct_pwrigid(save_movie=True)
        m_els = cm.load(mc.fname_tot_els)
        bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)),
                                     np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
        fnames = mc.fname_tot_els  # name of the pw-rigidly corrected file.

    else:
        mc.motion_correct_rigid(save_movie=True)
        m_els = cm.load(mc.fname_tot_rig)
        bord_px_els = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
        fnames = mc.fname_tot_rig  # name of the rigidly corrected file.

    # maximum shift to be used for trimming against NaNs
#%% compare with original movie
    if display_images:
        downsample_ratio = 0.2
        moviehandle = cm.concatenate([m_orig.resize(1, 1, downsample_ratio) - min_mov,
                                      m_els.resize(1, 1, downsample_ratio)],
                                     axis=2)
        moviehandle.play(fr=60, q_max=99.5, magnification=2)  # press q to exit

#%% MEMORY MAPPING
    # memory map the file in order 'C'
    border_to_0 = bord_px_els  # exclude borders due to motion correction
#    border_to_0 = 0 if mc.border_nan is 'copy' else bord_px_els   
        # you can include boundaries if you used the 'copy' option in the motion
        # correction, although be careful abou the components near the boundaries
    fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C',
                               border_to_0=border_to_0)  # exclude borders

    # now load the file
    Yr, dims, T = cm.load_memmap(fname_new)
    images = np.reshape(Yr.T, [T] + list(dims), order='F')
    # load frames in python format (T x X x Y)

#%% restart cluster to clean up memory
    cm.stop_server(dview=dview)
    c, dview, n_processes = cm.cluster.setup_cluster(
        backend='local', n_processes=None, single_thread=False)

#%%  parameters for source extraction and deconvolution
    p = 1                       # order of the autoregressive system
    gnb = 2                     # number of global background components
    merge_thresh = 0.8          # merging threshold, max correlation allowed
    # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
    rf = 15
    stride_cnmf = 6             # amount of overlap between the patches in pixels
    K = 4                       # number of components per patch
    gSig = [4, 4]               # expected half size of neurons
    # initialization method (if analyzing dendritic data using 'sparse_nmf')
    method_init = 'greedy_roi'

    # parameters for component evaluation

    opts = params.CNMFParams(dims=dims, fr=fr, decay_time=decay_time,
                             method_init=method_init, gSig=gSig,
                             merge_thresh=merge_thresh, p=p, gnb=gnb, k=K,
                             rf=rf, stride=stride_cnmf, rolling_sum=True)

#%% RUN CNMF ON PATCHES

    # First extract spatial and temporal components on patches and combine them
    # for this step deconvolution is turned off (p=0)

    opts.set('temporal', {'p': 0})
    cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
    cnm = cnm.fit(images)

#%% plot contours of found components
    Cn = cm.local_correlations(images.transpose(1, 2, 0))
    Cn[np.isnan(Cn)] = 0
    cnm.estimates.plot_contours(img=Cn)
    plt.title('Contour plots of found components')

#%% COMPONENT EVALUATION
    # the components are evaluated in three ways:
    #   a) the shape of each component must be correlated with the data
    #   b) a minimum peak SNR is required over the length of a transient
    #   c) each shape passes a CNN based classifier
    min_SNR = 2.5       # signal to noise ratio for accepting a component
    rval_thr = 0.8      # space correlation threshold for accepting a component
    cnn_thr = 0.8       # threshold for CNN based classifier
    cnm.params.set('quality', {'fr': fr,
                               'decay_time': decay_time,
                               'min_SNR': min_SNR,
                               'rval_thr': rval_thr,
                               'use_cnn': True,
                               'min_cnn_thr': cnn_thr})
    cnm.estimates.evaluate_components(images, cnm.params, dview=dview)

#%% PLOT COMPONENTS
    cnm.estimates.plot_contours(img=Cn, idx=cnm.estimates.idx_components)

#%% VIEW TRACES (accepted and rejected)

    if display_images:
        cnm.estimates.view_components(images, img=Cn, idx=cnm.estimates.idx_components)
        cnm.estimates.view_components(images, img=Cn, idx=cnm.estimates.idx_components_bad)

#%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution

    cnm.dview = None
    cnm2 = deepcopy(cnm)
    cnm2.dview = dview
    cnm2.params.set('patch', {'rf': None})
    cnm2.params.set('temporal', {'p': p})
    cnm2 = cnm2.fit(images)

#%% Extract DF/F values
    cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)

#%% Show final traces
    cnm2.estimates.view_components(Yr, img=Cn)

#%% reconstruct denoised movie (press q to exit)
    if display_images:
        cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2,
                                  magnification=2,
                                  bpx=border_to_0,
                                  include_bck=True)

#%% STOP CLUSTER and clean up log files
    cm.stop_server(dview=dview)
    log_files = glob.glob('*_LOG_*')
    for log_file in log_files:
        os.remove(log_file)
Example #14
0
def main():
#%% First setup some parameters for data and motion correction

    # dataset dependent parameters
    fr = 15.49             # imaging rate in frames per second
    decay_time = 0.9    # length of a typical transient in seconds
    dxy = (1.4, 1.4)      # spatial resolution in x and y in (um per pixel)
    # note the lower than usual spatial resolution here
    max_shift_um = (12., 12.)       # maximum shift in um
    patch_motion_um = (100., 100.)  # patch size for non-rigid correction in um

    # motion correction parameters
    pw_rigid = True       # flag to select rigid vs pw_rigid motion correction
    # maximum allowed rigid shift in pixels
    max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
    # start a new patch for pw-rigid motion correction every x pixels
    strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
    # overlap between pathes (size of patch in pixels: strides+overlaps)
    overlaps = (24, 24)
    # maximum deviation allowed for patch with respect to rigid shifts
    max_deviation_rigid = 3

    mc_dict = {
        'fnames': fnames,
        'fr': fr,
        'decay_time': decay_time,
        'dxy': dxy,
        'pw_rigid': pw_rigid,
        'max_shifts': max_shifts,
        'strides': strides,
        'overlaps': overlaps,
        'max_deviation_rigid': max_deviation_rigid,
        'border_nan': 'copy'
    }

    opts = params.CNMFParams(params_dict=mc_dict)

# %% start a cluster for parallel processing
    c, dview, n_processes = cm.cluster.setup_cluster(
        backend='local', n_processes=None, single_thread=False)

# %%% MOTION CORRECTION
    # first we create a motion correction object with the specified parameters
    mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
    # note that the file is not loaded in memory

# %% Run (piecewise-rigid motion) correction using NoRMCorre
    mc.motion_correct(save_movie=True)

# %% MEMORY MAPPING
    border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
    # you can include the boundaries of the FOV if you used the 'copy' option
    # during motion correction, although be careful about the components near
    # the boundaries

    # memory map the file in order 'C'
    fname_new = cm.save_memmap(mc.mmap_file, base_name='_memmap_', order='C',
                               border_to_0=border_to_0)  # exclude borders

    # now load the file
    Yr, dims, T = cm.load_memmap(fname_new)
    images = np.reshape(Yr.T, [T] + list(dims), order='F')
    # load frames in python format (T x X x Y)

# %% restart cluster to clean up memory
    cm.stop_server(dview=dview)
    c, dview, n_processes = cm.cluster.setup_cluster(
        backend='local', n_processes=None, single_thread=False)

# %%  parameters for source extraction and deconvolution
    p = 1                    # order of the autoregressive system
    gnb = 3                  # number of global background components
    merge_thr = 0.85         # merging threshold, max correlation allowed
    rf = 20                  # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
    stride_cnmf = 6          # amount of overlap between the patches in pixels
    K = 3                    # number of components per patch
    gSig = [7, 7]            # expected half size of neurons in pixels
    method_init = 'greedy_roi'   # initialization method (if analyzing dendritic data using 'sparse_nmf')
    ssub = 2                     # spatial subsampling during initialization
    tsub = 2                     # temporal subsampling during intialization
    s_min = -20                   # minimum signal amplitude needed in order for a transient to be considered as activity

    # parameters for component evaluation
    opts_dict = {'fnames': fnames,
                 'fr': fr,
                 'nb': gnb,
                 'rf': rf,
                 'K': K,
                 'gSig': gSig,
                 'stride': stride_cnmf,
                 'method_init': method_init,
                 'rolling_sum': True,
                 'merge_thr': merge_thr,
                 'n_processes': n_processes,
                 'only_init': True,
                 'ssub': ssub,
                 'tsub': tsub,
                 's_min': s_min}

    opts.change_params(params_dict=opts_dict)

# %% RUN CNMF ON PATCHES
    # First extract spatial and temporal components on patches and combine them
    # for this step deconvolution is turned off (p=0)
    opts.change_params({'p': 0})
    cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
    cnm = cnm.fit(images)

# %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
    #   you can also perform the motion correction plus cnmf fitting steps
    #   simultaneously after defining your parameters object using
    #  cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
    #  cnm1.fit_file(motion_correct=True)


# %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
    cnm.params.change_params({'p': p})
    cnm2 = cnm.refit(images, dview=dview)
    # %% COMPONENT EVALUATION
    # the components are evaluated in three ways:
    #   a) the shape of each component must be correlated with the data
    #   b) a minimum peak SNR is required over the length of a transient
    #   c) each shape passes a CNN based classifier
    min_SNR = 2  # Overall minimum signal to noise ratio for accepting a component
    rval_thr = 0.85  # space correlation threshold for accepting a component
    cnn_thr = 0.99  # threshold for CNN based classifier
    cnn_lowest = 0.15 # neurons with cnn probability lower than this value are rejected
    min_size_neuro = 0.1*gSig[0]*np.pi**2
    max_size_neuro = 2.5*gSig[0]*np.pi**2

    cnm2.params.set('quality', {'decay_time': decay_time,
                               'min_SNR': min_SNR,
                               'rval_thr': rval_thr,
                               'use_cnn': True,
                               'min_cnn_thr': cnn_thr,
                               'cnn_lowest': cnn_lowest,
                               'min_size_neuro': min_size_neuro,
                               'max_size_neuro': max_size_neuro,})

    cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)

    #%% update object with selected components
    cnm2.estimates.select_components(use_object=True)

    #%% Extract DF/F values
    cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)


    #cnm2.estimates.threshold_spatial_components(maxthr=0.85)
    #cnm2.estimates.remove_small_large_neurons(min_size_neuro=min_size_neuro, max_size_neuro=max_size_neuro)

    # %% Generate heat image of video and save data
    Cn = cm.local_correlations(images, swap_dim=False)
    Cn[np.isnan(Cn)] = 0
    cnm2.estimates.Cn = Cn
    cnm2.save(cnm2.mmap_file[:-4] + 'hdf5')


    #%% STOP CLUSTER and clean up log files
    cm.stop_server(dview=dview)

    #Create denoised video as matrix
    denoised = cm.movie(cnm2.estimates.A.dot(cnm2.estimates.C) + \
                    cnm2.estimates.b.dot(cnm2.estimates.f)).reshape(dims + (-1,), order='F').transpose([2, 0, 1])

    #Normalizing denoised matrix to the [0,255] range prior to type conversion
    max_pixel = np.nanmax(denoised)
    norm_denoised = (denoised/max_pixel)*255

    #Type conversion so imageio doesn't get mad
    norm_denoised = np.uint8(norm_denoised)

    #Saving a denoised avi of the analyzed video
    sname = Path(args.in_file).stem + '.avi'
    imageio.mimwrite(sname, norm_denoised, fps=15.49)
def initialize_movie(Y, K, gSig, rf, stride, base_name,
                     p=1, merge_thresh=0.95, rval_thr_online=0.9, thresh_fitness_delta_online=-30, thresh_fitness_raw_online=-50,
                     rval_thr_init=.5, thresh_fitness_delta_init=-20, thresh_fitness_raw_init=-20,
                     rval_thr_refine=0.95, thresh_fitness_delta_refine=-100, thresh_fitness_raw_refine=-100,
                     final_frate=10, Npeaks=10, single_thread=True, dview=None):

    _, d1, d2 = Y.shape
    dims = (d1, d2)
    Yr = Y.to_2D().T
    # merging threshold, max correlation allowed
    # order of the autoregressive system
    #T = Y.shape[0]
    base_name = base_name + '.mmap'
    fname_new = Y.save(base_name, order='C')
    #%
    Yr, dims, T = cm.load_memmap(fname_new)
    d1, d2 = dims
    images = np.reshape(Yr.T, [T] + list(dims), order='F')
    Y = np.reshape(Yr, dims + (T,), order='F')
    Cn2 = cm.local_correlations(Y)
#    pl.imshow(Cn2)
    #%
    #% RUN ALGORITHM ON PATCHES
#    pl.close('all')
    cnm_init = cnmf.CNMF(n_processes, method_init='greedy_roi', k=K, gSig=gSig, merge_thresh=merge_thresh,
                         p=0, dview=dview, Ain=None, rf=rf, stride=stride, method_deconvolution='oasis', skip_refinement=False,
                         normalize_init=False, options_local_NMF=None,
                         minibatch_shape=100, minibatch_suff_stat=5,
                         update_num_comps=True, rval_thr=rval_thr_online, thresh_fitness_delta=thresh_fitness_delta_online, thresh_fitness_raw=thresh_fitness_raw_online,
                         batch_update_suff_stat=True, max_comp_update_shape=5)

    cnm_init = cnm_init.fit(images)
    A_tot = cnm_init.A
    C_tot = cnm_init.C
    YrA_tot = cnm_init.YrA
    b_tot = cnm_init.b
    f_tot = cnm_init.f

    print(('Number of components:' + str(A_tot.shape[-1])))

    #%

    traces = C_tot + YrA_tot
    #        traces_a=traces-scipy.ndimage.percentile_filter(traces,8,size=[1,np.shape(traces)[-1]/5])
    #        traces_b=np.diff(traces,axis=1)
    fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = evaluate_components(
        Y, traces, A_tot, C_tot, b_tot, f_tot, final_frate, remove_baseline=True, N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks,  thresh_C=0.3)

    idx_components_r = np.where(r_values >= rval_thr_init)[0]
    idx_components_raw = np.where(fitness_raw < thresh_fitness_raw_init)[0]
    idx_components_delta = np.where(
        fitness_delta < thresh_fitness_delta_init)[0]

    idx_components = np.union1d(idx_components_r, idx_components_raw)
    idx_components = np.union1d(idx_components, idx_components_delta)
    idx_components_bad = np.setdiff1d(list(range(len(traces))), idx_components)

    print(('Keeping ' + str(len(idx_components)) +
           ' and discarding  ' + str(len(idx_components_bad))))

    A_tot = A_tot.tocsc()[:, idx_components]
    C_tot = C_tot[idx_components]
    #%
    cnm_refine = cnmf.CNMF(n_processes, method_init='greedy_roi', k=A_tot.shape, gSig=gSig, merge_thresh=merge_thresh, rf=None, stride=None,
                           p=p, dview=dview, Ain=A_tot, Cin=C_tot, f_in=f_tot, method_deconvolution='oasis', skip_refinement=True,
                           normalize_init=False, options_local_NMF=None,
                           minibatch_shape=100, minibatch_suff_stat=5,
                           update_num_comps=True, rval_thr=rval_thr_refine, thresh_fitness_delta=thresh_fitness_delta_refine, thresh_fitness_raw=thresh_fitness_raw_refine,
                           batch_update_suff_stat=True, max_comp_update_shape=5)

    cnm_refine = cnm_refine.fit(images)
    #%
    A, C, b, f, YrA, sn = cnm_refine.A, cnm_refine.C, cnm_refine.b, cnm_refine.f, cnm_refine.YrA, cnm_refine.sn
    #%
    final_frate = 10
    Npeaks = 10
    traces = C + YrA

    fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = \
        evaluate_components(Y, traces, A, C, b, f, final_frate, remove_baseline=True,
                            N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks,  thresh_C=0.3)

    idx_components_r = np.where(r_values >= rval_thr_refine)[0]
    idx_components_raw = np.where(fitness_raw < thresh_fitness_raw_refine)[0]
    idx_components_delta = np.where(
        fitness_delta < thresh_fitness_delta_refine)[0]

    idx_components = np.union1d(idx_components_r, idx_components_raw)
    idx_components = np.union1d(idx_components, idx_components_delta)
    idx_components_bad = np.setdiff1d(list(range(len(traces))), idx_components)

    print(' ***** ')
    print((len(traces)))
    print((len(idx_components)))
    #%
    cnm_refine.idx_components = idx_components
    cnm_refine.idx_components_bad = idx_components_bad
    cnm_refine.r_values = r_values
    cnm_refine.fitness_raw = fitness_raw
    cnm_refine.fitness_delta = fitness_delta
    cnm_refine.Cn2 = Cn2

    #%

#    cnm_init.dview = None
#    save_object(cnm_init,fls[0][:-4]+ '_DS_' + str(ds)+ '_init.pkl')

    return cnm_refine, Cn2, fname_new
Example #16
0
def main():
    pass # For compatibility between running under Spyder and the CLI

#%% First setup some parameters

    # num processes
    n_proc = 12
    # dataset dependent parameters
    fr = 30                             # imaging rate in frames per second
    decay_time = 0.4                    # length of a typical transient in seconds

    # motion correction parameters
    niter_rig = 1               # number of iterations for rigid motion correction
    max_shifts = (6, 6)         # maximum allow rigid shift
    # for parallelization split the movies in  num_splits chuncks across time
    splits_rig = 56
    # start a new patch for pw-rigid motion correction every x pixels
    strides = (48, 48)
    # overlap between pathes (size of patch strides+overlaps)
    overlaps = (24, 24)
    # for parallelization split the movies in  num_splits chuncks across time
    splits_els = 56
    upsample_factor_grid = 4    # upsample factor to avoid smearing when merging patches
    # maximum deviation allowed for patch with respect to rigid shifts
    max_deviation_rigid = 3

    # parameters for source extraction and deconvolution
    p = 1                       # order of the autoregressive system
    gnb = 2                     # number of global background components
    merge_thresh = 0.8          # merging threshold, max correlation allowed
    # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
    rf = 15
    stride_cnmf = 6             # amount of overlap between the patches in pixels
    K = 4                       # number of components per patch
    gSig = [4, 4]               # expected half size of neurons
    # initialization method (if analyzing dendritic data using 'sparse_nmf')
    init_method = 'greedy_roi'
    is_dendrites = False        # flag for analyzing dendritic data
    # sparsity penalty for dendritic data analysis through sparse NMF
    alpha_snmf = None

    # parameters for component evaluation
    min_SNR = 2.5               # signal to noise ratio for accepting a component
    rval_thr = 0.8              # space correlation threshold for accepting a component
    cnn_thr = 0.8               # threshold for CNN based classifier


    
#%% start a cluster for parallel processing
    c, dview, n_processes = cm.cluster.setup_cluster(
        backend='local', n_processes=n_proc, single_thread=False)
    
    print('Parallel processing initialized.')
    print('Beginning motion correction')

#%%% MOTION CORRECTION

    t_ms = time.time()
    # first we create a motion correction object with the parameters specified
    min_mov = cm.load(fname[0], subindices=range(200)).min()
    # this will be subtracted from the movie to make it non-negative
    if not nomc:
      mc = MotionCorrect(fname[0], min_mov,
                         dview=dview, max_shifts=max_shifts, niter_rig=niter_rig,splits_rig=splits_rig,strides=strides, overlaps=overlaps, splits_els=splits_els,upsample_factor_grid=upsample_factor_grid,max_deviation_rigid=max_deviation_rigid,shifts_opencv=True, nonneg_movie=True, use_cuda=use_cuda)
		# note that the file is not loaded in memory

		#%% Run piecewise-rigid motion correction using NoRMCorre
      mc.motion_correct_pwrigid(save_movie=True)
      m_els = cm.load(mc.fname_tot_els)
      bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)),np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
      t_mf = time.time()
      print('Motion correction complete in ', int(t_mf - t_ms),' seconds')
		# maximum shift to be used for trimming against NaNs
		#%% compare with original movie
    

#%% MEMORY MAPPING
    # memory map the file in order 'C'
      fnames = mc.fname_tot_els   # name of the pw-rigidly corrected file.
      border_to_0 = bord_px_els     # number of pixels to exclude
      fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C',
			       border_to_0=bord_px_els)  # exclude borders


    bord_px_els = 5
    # now load the file
    Yr, dims, T = cm.load_memmap(fname_new)
    d1, d2 = dims
    images = np.reshape(Yr.T, [T] + list(dims), order='F')
    # load frames in python format (T x X x Y)

#%% restart cluster to clean up memory
    cm.stop_server(dview=dview)
    c, dview, n_processes = cm.cluster.setup_cluster(
        backend='local', n_processes=n_proc, single_thread=False)

#%% RUN CNMF ON PATCHES

    # First extract spatial and temporal components on patches and combine them
    # for this step deconvolution is turned off (p=0)
    print('Beginning initial CNMF fit')
    
    t1 = time.time()

    cnm = cnmf.CNMF(n_processes=n_proc, k=K, gSig=gSig, merge_thresh=merge_thresh,
                    p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1,
                    method_init=init_method, alpha_snmf=alpha_snmf,
                    only_init_patch=False, gnb=gnb, border_pix=bord_px_els)
    cnm = cnm.fit(images)
    
    t2 = time.time()
    print('Initial CNMF fit complete in ', int(t2-t1), 'seconds.')
    

    Cn = cm.local_correlations(images.transpose(1, 2, 0))
    Cn[np.isnan(Cn)] = 0

#%% COMPONENT EVALUATION
    # the components are evaluated in three ways:
    #   a) the shape of each component must be correlated with the data
    #   b) a minimum peak SNR is required over the length of a transient
    #   c) each shape passes a CNN based classifier
    t3 = time.time()
    print('Estimating component quality')
    
    idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
        estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
                                         cnm.YrA, fr, decay_time, gSig, dims,
                                         dview=dview, min_SNR=min_SNR,
                                         r_values_min=rval_thr, use_cnn=False,
                                         thresh_cnn_min=cnn_thr)
    t4 = time.time()
    print('Component quality estimation complete in ', int(t4-t3),' seconds')


#%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
    print('Re-running CNMF to refine and deconvolve')
    t5 = time.time()
    
    A_in, C_in, b_in, f_in = cnm.A[:,
                                   idx_components], cnm.C[idx_components], cnm.b, cnm.f
    cnm2 = cnmf.CNMF(n_processes=n_proc, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview,
                     merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in,
                     f_in=f_in, rf=None, stride=None, gnb=gnb,
                     method_deconvolution='oasis', check_nan=True)

    cnm2 = cnm2.fit(images)
    t6 = time.time()
    print('CNMF re-run complete in ', int(t6-t5), ' seconds')
#%% Extract DF/F values

#    F_dff = detrend_df_f(cnm2.A, cnm2.b, cnm2.C, cnm2.f, YrA=cnm2.YrA,
#                         quantileMin=8, frames_window=250)


    save_results = True
    if save_results:
        if os.path.isdir(infile[0]):
          outfile = os.path.join(infile[0],'caiman_output.npz')
        else:
          outfile = os.path.join(os.path.split(infile[0])[0],'caiman_output.npz')
        np.savez_compressed(outfile,Cn=Cn, A=cnm2.A.todense(), C=cnm2.C,b=cnm2.b, f=cnm2.f, YrA=cnm2.YrA, d1=d1, d2=d2,idx_components=idx_components, idx_components_bad=idx_components_bad)

    
#%% STOP CLUSTER and clean up log files
    cm.stop_server(dview=dview)
    fname_new = cm.save_memmap_join(name_new, base_name='Yr', n_chunks=12, dview=dview)
else:
    fname_new = name_new[0]
#%%
from time import time    
# fname_new='Yr_d1_501_d2_398_d3_1_order_F_frames_369_.mmap'
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
Y = np.reshape(Yr, dims + (T,), order='F')
#%%
#if np.min(images)<0:
#    raise Exception('Movie too negative, add_to_movie should be larger')
#%%
if 0:
    Cn = cm.local_correlations(Y[:,:,::np.int(old_div(np.shape(images)[0],3000))]) 
    pl.imshow(Cn)  

#%%
#if not is_patches:
#    #%%
#    K = 35  # number of neurons expected per patch
#    gSig = [7, 7]  # expected half size of neurons
#    merge_thresh = 0.8  # merging threshold, max correlation allowed
#    p = 2  # order of the autoregressive system
#    cnm = cnmf.CNMF(n_processes, method_init=init_method, k=K, gSig=gSig, merge_thresh=merge_thresh,
#                    p=p, dview=dview, Ain=None,method_deconvolution='oasis')
#    cnm = cnm.fit(images)
#
##%%
#else:
m1.save('MOV_EXAMPLE_20160706154257.tif')
#%%
name_new = cm.save_memmap_each(
    ['MOV_EXAMPLE_20160706154257.tif'], dview=c[:], base_name=None)
#%%
n_chunks = 6  # increase this number if you have memory issues at this point
fname_new = cm.save_memmap_join(
    name_new, base_name='MOV_EXAMPLE_20160706154257__', n_chunks=6, dview=dview)
#%%
Yr, dims, T = cm.load_memmap(
    'MOV_EXAMPLE_20160706154257___d1_150_d2_330_d3_1_order_C_frames_2300_.mmap')
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
Y = np.reshape(Yr, dims + (T,), order='F')
#%%
Cn = cm.local_correlations(Y[:, :, :3000], swap_dim=True)
pl.imshow(Cn, cmap='gray')
#%%
rf = 10  # half-size of the patches in pixels. rf=25, patches are 50x50
stride = 4  # amounpl.it of overlap between the patches in pixels
K = 4  # number of neurons expected per patch
gSig = [5, 5]  # expected half size of neurons
merge_thresh = 0.8  # merging threshold, max correlation allowed
p = 2  # order of the autoregressive system
memory_fact = 1  # unitless number accounting how much memory should be used. You will need to try different values to see which one would work the default is OK for a 16 GB system
save_results = False
#%% RUN ALGORITHM ON PATCHES
cnm = cnmf.CNMF(n_processes, k=K, gSig=gSig, merge_thresh=0.8, p=0, dview=dview, Ain=None,
                rf=rf, stride=stride, memory_fact=memory_fact,
                method_init='greedy_roi', alpha_snmf=10e2)
cnm = cnm.fit(images)
plt.savefig(os.path.join(root, 'all_components.png'))
plt.close()

#%% serial evaluation
root = r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch3\M32\20200318'

#%% perform extraction
motion_file, dview = pipe.motion_correction(root, opts, dview, remove_f_order=True, remove_c_order=True)

root = r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch3\M32\20200322'

mmap_file, images = pipe.load_mmap(root)
cnm_params = cnm_params.change_params({'fnames': mmap_file[0]})
cnm = pipe.run_source_extraction(images, cnm_params, dview=dview)
pipe.save_cnmf(cnm, path=os.path.join(root, 'cnm_pre_selection.hdf5'), verbose=False, overwrite=True)
lcm = cm.local_correlations(images, swap_dim=False)
lcm[np.isnan(lcm)] = 0
cnm.estimates.Cn = lcm
pipe.save_cnmf(cnm, path=os.path.join(root, 'cnm_pre_selection.hdf5'), verbose=False, overwrite=True)
cnm.estimates.plot_contours(img=cnm.estimates.Cn, display_numbers=False)
plt.tight_layout()
fig = plt.gcf()
fig.set_size_inches((10, 10))
plt.savefig(os.path.join(root, 'pre_sel_components.png'))
pipe.save_cnmf(cnm, path=os.path.join(root, 'cnm_pre_selection.hdf5'), verbose=False, overwrite=True)
plt.close()


#%% load pre-extracted data
cnm = pipe.load_cnmf(root, 'cnm_pre_selection.hdf5')
mmap_file, images = pipe.load_mmap(root)
Example #20
0
#idx_xy=(idx_x,idx_y)
add_to_movie=0 # of movie too negative need to add a baseline
downsample_factor=1 # use .2 or .1 if file is large and you want a quick answer
idx_xy=None
base_name='Yr'
name_new=cm.save_memmap_each(fnames, dview=dview,base_name=base_name, resize_fact=(1, 1, downsample_factor), remove_init=0,idx_xy=idx_xy,add_to_movie=add_to_movie )
name_new.sort()
print(name_new)

#%%
fname_new=cm.save_memmap_join(name_new,base_name='Yr', n_chunks=12, dview=dview)
#%%
Yr,dims,T=cm.load_memmap(fname_new)
Y=np.reshape(Yr,dims+(T,),order='F')
#%% visualize correlation image
Cn = cm.local_correlations(Y)
pl.imshow(Cn,cmap='gray')   
#%% parameters of experiment
K=30 # number of neurons expected per patch
gSig=[7,7] # expected half size of neurons
merge_thresh=0.8 # merging threshold, max correlation allowed
p=2 #order of the autoregressive system
options = cnmf.utilities.CNMFSetParms(Y,n_processes,p=p,gSig=gSig,K=K,ssub=2,tsub=2,nb=1)
options['preprocess_params']['noise_method']='mean'
#%% PREPROCESS DATA AND INITIALIZE COMPONENTS
t1 = time()
Yr,sn,g,psx = cm.source_extraction.cnmf.pre_processing.preprocess_data(Yr,dview=dview,**options['preprocess_params'])
print((time() - t1))
#%%
t1 = time()
Atmp, Ctmp, b_in, f_in, center=cm.source_extraction.cnmf.initialization.initialize_components(Y, normalize=True, **options['init_params'])                                                    
def main():
    pass  # For compatibility between running under Spyder and the CLI

#%% Select file(s) to be processed (download if not present)
    root = '/Users/hheiser/Desktop/testing data/chronic_M2N3/0d_baseline/channel1'
    fnames = [r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\1\file_00003.tif',
              r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\2\file_00004.tif',
              r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\3\file_00005.tif',
              r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\4\file_00006.tif',
              r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\5\file_00007.tif',
              r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\6\file_00008.tif']

    fnames = ['Sue_2x_3000_40_-46.tif']  # filename to be processed
    if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
        fnames = [download_demo(fnames[0])]

#%% First setup some parameters for data and motion correction

    # dataset dependent parameters
    fr = 30             # imaging rate in frames per second
    decay_time = 0.4    # length of a typical transient in seconds
    dxy = (2., 2.)      # spatial resolution in x and y in (um per pixel)
    # note the lower than usual spatial resolution here
    max_shift_um = (12., 12.)       # maximum shift in um
    patch_motion_um = (100., 100.)  # patch size for non-rigid correction in um

    # motion correction parameters
    pw_rigid = True       # flag to select rigid vs pw_rigid motion correction
    # maximum allowed rigid shift in pixels
    max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
    # start a new patch for pw-rigid motion correction every x pixels
    strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
    # overlap between pathes (size of patch in pixels: strides+overlaps)
    overlaps = (24, 24)
    # maximum deviation allowed for patch with respect to rigid shifts
    max_deviation_rigid = 3

    mc_dict = {
        'fnames': fnames,
        'fr': fr,
        'decay_time': decay_time,
        'dxy': dxy,
        'pw_rigid': pw_rigid,
        'max_shifts': max_shifts,
        'strides': strides,
        'overlaps': overlaps,
        'max_deviation_rigid': max_deviation_rigid,
        'border_nan': 'copy'
    }

    opts = params.CNMFParams(params_dict=mc_dict)

#%% play the movie (optional)
    # playing the movie using opencv. It requires loading the movie in memory.
    # To close the video press q
    display_images = True

    if display_images:
        m_orig = cm.load_movie_chain(fnames)
        ds_ratio = 0.2
        moviehandle = m_orig.resize(1, 1, ds_ratio)
        moviehandle.play(q_max=99.5, fr=30, magnification=1, do_loop=False)

#%% start a cluster for parallel processing
    c, dview, n_processes = cm.cluster.setup_cluster(
        backend='local', n_processes=None, single_thread=False)

#%% MOTION CORRECTION
    # first we create a motion correction object with the specified parameters
    mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
    # note that the file is not loaded in memory

#%% Run (piecewise-rigid motion) correction using NoRMCorre
    mc.motion_correct(save_movie=True)

#%% compare with original movie
    if display_images:
        m_orig = cm.load_movie_chain(fnames[:3])
        m_els = cm.load(mmap_file[:3])
        ds_ratio = 0.2
        moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie,
                                      m_els.resize(1, 1, ds_ratio)], axis=2)
        moviehandle.play(fr=15, q_max=99.5, magnification=2)  # press q to exit

#%% MEMORY MAPPING
    border_to_0 = 0 if mc.border_nan == 'copy' else mc.border_to_0
    # you can include the boundaries of the FOV if you used the 'copy' option
    # during motion correction, although be careful about the components near
    # the boundaries

    mmap_file = [r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\1\file_00001_els__d1_512_d2_512_d3_1_order_F_frames_1147_.mmap',
                 r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\2\file_00002_els__d1_512_d2_512_d3_1_order_F_frames_2520_.mmap',
                 r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\3\file_00003_els__d1_512_d2_512_d3_1_order_F_frames_3814_.mmap',
                 r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\4\file_00004_els__d1_512_d2_512_d3_1_order_F_frames_5154_.mmap',
                 r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\5\file_00005_els__d1_512_d2_512_d3_1_order_F_frames_2677_.mmap',
                 r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\6\file_00006_els__d1_512_d2_512_d3_1_order_F_frames_3685_.mmap']

    fname_new = r'E:\PhD\Data\DG\M14_20191014\N1\memmap__d1_512_d2_512_d3_1_order_C_frames_34939_.mmap'

    # memory map the file in order 'C'
    fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C',
                               border_to_0=0)  # exclude borders

    # now load the file
    Yr, dims, T = cm.load_memmap(fname_new)
    images = np.reshape(Yr.T, [T] + list(dims), order='F')
    # load frames in python format (T x X x Y)


#%% restart cluster to clean up memory
    cm.stop_server(dview=dview)
    c, dview, n_processes = cm.cluster.setup_cluster(
        backend='local', n_processes=None, single_thread=False)

#%%  parameters for source extraction and deconvolution
    p = 1                    # order of the autoregressive system
    gnb = 2                  # number of global background components
    merge_thr = 0.85         # merging threshold, max correlation allowed
    rf = 15
    # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
    stride_cnmf = 6          # amount of overlap between the patches in pixels
    K = 4                    # number of components per patch
    gSig = [4, 4]            # expected half size of neurons in pixels
    # initialization method (if analyzing dendritic data using 'sparse_nmf')
    method_init = 'greedy_roi'
    ssub = 2                     # spatial subsampling during initialization
    tsub = 2                     # temporal subsampling during intialization

    # parameters for component evaluation
    opts_dict = {'fnames': fnames,
                 'fr': fr,
                 'nb': gnb,
                 'rf': rf,
                 'K': K,
                 'gSig': gSig,
                 'stride': stride_cnmf,
                 'method_init': method_init,
                 'rolling_sum': True,
                 'merge_thr': merge_thr,
                 'n_processes': n_processes,
                 'only_init': True,
                 'ssub': ssub,
                 'tsub': tsub}

    opts.change_params(params_dict=opts_dict)
#%% RUN CNMF ON PATCHES
    # First extract spatial and temporal components on patches and combine them
    # for this step deconvolution is turned off (p=0)

    #opts.change_params({'p': 1,'rf':None, 'only_init':False})
    opts.change_params({'p': 0})
    cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
    cnm = cnm.fit(images)

#%% RUN CNMF SEEDED WITH MANUAL MASK

    # load mask
    mask = np.asarray(imageio.imread('/Users/hheiser/Desktop/testing data/file_00020_no_motion/avg_mask_fixed.png'), dtype=bool)

    # get component ROIs from the mask and plot them
    Ain, labels, mR = cm.base.rois.extract_binary_masks(mask)

    # plot original mask and extracted labels to check mask
    fig, ax = plt.subplots(1,2)
    ax[0].imshow(-mR,cmap='binary')
    ax[0].set_title('Original mask')
    ax[1].imshow(labels)
    ax[1].set_title('Extracted labelled ROIs')

    """"
    plt.figure()
    crd = cm.utils.visualization.plot_contours(
        Ain.astype('float32'), mR, thr=0.99, display_numbers=True) # todo check if this is important for the pipeline
    plt.title('Contour plots of detected ROIs in the structural channel')
    """

    opts.change_params({'rf': None, 'only_init': False})

    # run CNMF seeded with this mask
    cnm_corr = cnmf.CNMF(n_processes, params=opts, dview=dview, Ain=Ain)
    cnm_corr_seed = cnm_corr_seed.fit(images)
    #cnm_seed = cnm_seed.fit_file(motion_correct=False)

#%% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
    #   you can also perform the motion correction plus cnmf fitting steps
    #   simultaneously after defining your parameters object using
    #  cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
    #  cnm1.fit_file(motion_correct=True)

#%% plot contours of found components
    Cn = cm.local_correlations(images, swap_dim=False)
    Cn[np.isnan(Cn)] = 0
    cnm.estimates.plot_contours(img=Cn)
    plt.title('Contour plots of found components')

#%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
    cnm.params.change_params({'p': p})
    cnm2 = cnm.refit(images, dview=dview)

#%% COMPONENT EVALUATION
    # the components are evaluated in three ways:
    #   a) the shape of each component must be correlated with the data
    #   b) a minimum peak SNR is required over the length of a transient
    #   c) each shape passes a CNN based classifier
    min_SNR = 2  # signal to noise ratio for accepting a component
    rval_thr = 0.85  # space correlation threshold for accepting a component
    cnn_thr = 0.99  # threshold for CNN based classifier
    cnn_lowest = 0.1  # neurons with cnn probability lower than this value are rejected

    cnm2.params.set('quality', {'decay_time': decay_time,
                               'min_SNR': min_SNR,
                               'rval_thr': rval_thr,
                               'use_cnn': True,
                               'min_cnn_thr': cnn_thr,
                               'cnn_lowest': cnn_lowest})
    cnm2.estimates.evaluate_components(images, params=cnm2.params, dview=dview)

#%% PLOT COMPONENTS
    cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)

#%% VIEW TRACES (accepted and rejected)

    if display_images:
        cnm2.estimates.view_components(images, img=Cn,
                                      idx=cnm2.estimates.idx_components)
        cnm2.estimates.view_components(images, img=Cn,
                                      idx=cnm2.estimates.idx_components_bad)
#%% update object with selected components
    #### -> will delete rejected components!
    cnm2.estimates.select_components(use_object=True)
#%% Extract DF/F values
    cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)

#%% Show final traces
    cnm2.estimates.view_components(img=Cn)

#%% reconstruct denoised movie (press q to exit)
    if display_images:
        cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2,
                                  magnification=1,
                                  bpx=border_to_0,
                                  include_bck=True)  # background not shown

#%% STOP CLUSTER and clean up log files
    cm.stop_server(dview=dview)
    log_files = glob.glob('*_LOG_*')
    for log_file in log_files:
        os.remove(log_file)

#%% save results

    dirname = fnames[0][:-4] + "_results.hdf5"
    cnm2.estimates.Cn = Cn
    cnm2.save(dirname)

    #load results
    cnm2 = cnmf.load_CNMF(dirname)

    mov_name = fnames[0][:-4] + "_movie_restored_2_gain.tif"
    helper.save_movie(cnm2.estimates,images,mov_name,frame_range=range(200),include_bck=True)
Example #22
0
from copy import deepcopy
from scipy.special import log_ndtr

import sys, os

#********************************************************************************************************************************
#********************************************************************************************************************************
#********************************************************************************************************************************

#%%
#fname = './example_movies/demoMovie.tif'
fname = sys.argv[1] #'/mnt/244f644c-15b8-46be-8f8a-50b5b2d8c6e1/in_vivo/rafa/alejandro/G2M5/20170511/000/G2M5_C1V1_GCaMP6s_20170511_000.tif'
print ("...running demo_OnACID on file: ", fname)

Y = cm.load(fname).astype(np.float32)                   # 
Cn = cm.local_correlations(Y.transpose(1, 2, 0))        # used as a background image


#Load parameters from .npz file
if True:
    print (sys.argv[1][:-4]+"_runtime_params.npz")
    params = np.load(sys.argv[1][:-4]+"_runtime_params.npz")
    
    print (type(params['merge_thr']))
    merge_thresh=np.float32(params['merge_thr']); print (merge_thresh)
    initbatch = np.int32(params['initibatch']); print (initbatch)
    patch_size = np.int32(params['patch_size']); print (patch_size)
    stride=np.int32(params['stride']); print (stride)
    K=np.int32(params['K']); print (K)
    
    frame_rate=np.float32(params['frame_rate']); print (frame_rate)
Example #23
0
Created on Wed Sep 20 17:52:23 2017

@author: jfriedrich
"""

import numpy as np
import pylab as pl
import caiman as cm
from caiman.source_extraction import cnmf as cnmf
from caiman.utils.visualization import view_patches_bar, plot_contours
from copy import deepcopy

#%%
fname = './example_movies/demoMovie.tif'
Y = cm.load(fname).astype(np.float32)
Cn = cm.local_correlations(Y.transpose(1, 2, 0))

T1 = Y.shape[0]
merge_thresh = 0.8  # merging threshold, max correlation allowed
p = 1  # order of the autoregressive system
initbatch = 400
expected_comps = 50
rf = 16
stride = 3
K = 4
gSig = [6, 6]  # expected half size of neurons
rval_thr = 0.95
thresh_fitness_delta = -50
thresh_fitness_raw = -50

fname_new = Y[:initbatch].save('demo.mmap', order='C')
def main():
    pass  # For compatibility between running under Spyder and the CLI

    #%%
    """
    General parameters
    """
    play_movie = 1
    plot_extras = 1
    plot_extras_cell = 1
    compute_mc_metrics = 1

    #%% Select file(s) to be processed (download if not present)
    """
    Load file
    """

    #fnames = ['Sue_2x_3000_40_-46.tif']  # filename to be processed
    #if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
    #    fnames = [download_demo(fnames[0])]

    #fnames = ['/home/yuriy/Desktop/Data/rest1_5_9_19_cut.tif']

    #f_dir = 'C:\\Users\\rylab_dataPC\\Desktop\\Yuriy\\caiman_data\\short\\'
    f_dir = 'G:\\analysis\\190828-calcium_voltage\\soma_dendrites\\pCAG_jREGECO1a_ASAP3_anesth_001\\'
    f_name = 'Ch1'
    f_ext = 'tif'
    fnames = [f_dir + f_name + '.' + f_ext]

    #fnames = ['C:/Users/rylab_dataPC/Desktop/Yuriy/caiman_data/rest1_5_9_19_2_cut_ca.hdf5']

    #%% First setup some parameters for data and motion correction
    """
    Parameters
    """

    # dataset dependent parameters
    fr = 30  # imaging rate in frames per second
    decay_time = 1
    #0.4    # length of a typical transient in seconds
    dxy = (2., 2.)  # spatial resolution in x and y in (um per pixel)
    # note the lower than usual spatial resolution here
    max_shift_um = (12., 12.)  # maximum shift in um
    patch_motion_um = (100., 100.)  # patch size for non-rigid correction in um

    # motion correction parameters
    pw_rigid = True  # flag to select rigid vs pw_rigid motion correction
    # maximum allowed rigid shift in pixels
    #max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
    max_shifts = [6, 6]
    # start a new patch for pw-rigid motion correction every x pixels
    #strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
    strides = [48, 48]
    # overlap between pathes (size of patch in pixels: strides+overlaps)
    overlaps = (24, 24)
    # maximum deviation allowed for patch with respect to rigid shifts
    max_deviation_rigid = 3

    mc_dict = {
        'fnames': fnames,
        'fr': fr,
        'decay_time': decay_time,
        'dxy': dxy,
        'pw_rigid': pw_rigid,
        'max_shifts': max_shifts,
        'strides': strides,
        'overlaps': overlaps,
        'max_deviation_rigid': max_deviation_rigid,
        'border_nan': 'copy'
    }

    opts = params.CNMFParams(params_dict=mc_dict)

    # %% play the movie (optional)
    # playing the movie using opencv. It requires loading the movie in memory.
    # To close the video press q

    if play_movie:
        m_orig = cm.load_movie_chain(fnames)
        ds_ratio = 0.2
        moviehandle = m_orig.resize(1, 1, ds_ratio)
        moviehandle.play(q_max=99.5, fr=60, magnification=2)

    # %% start a cluster for parallel processing
    c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
                                                     n_processes=None,
                                                     single_thread=False)

    # %%% MOTION CORRECTION
    # first we create a motion correction object with the specified parameters
    mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
    # note that the file is not loaded in memory

    # %% Run (piecewise-rigid motion) correction using NoRMCorre
    mc.motion_correct(save_movie=True)

    # type "mc."and press TAB to see all interesting associated variables and self. outputs
    # interesting outputs
    # saved file is mc.fname_tot_els / mc.fname_tot_rig
    # mc.x_shifts_els / mc.y_shifts_els: shifts in x/y per frame per patch
    # mc.coord_shifts_els: coordinates associated to patches with shifts
    # mc.total_template_els: updated template for pw
    # mc.total_template_rig: updated template for rigid
    # mc.templates_rig: templates for each iteration in rig

    #%% # compute metrics for the results (TAKES TIME!!)
    if compute_mc_metrics:
        # not finished
        bord_px_els = np.ceil(
            np.maximum(np.max(np.abs(mc.x_shifts_els)),
                       np.max(np.abs(mc.y_shifts_els)))).astype(np.int)

        final_size = np.subtract(
            mc.total_template_els.shape,
            2 * bord_px_els)  # remove pixels in the boundaries
        winsize = 100
        swap_dim = False
        resize_fact_flow = .2  # downsample for computing ROF

        tmpl_rig, correlations_orig, flows_orig, norms_orig, crispness_orig = cm.motion_correction.compute_metrics_motion_correction(
            fnames[0],
            final_size[0],
            final_size[1],
            swap_dim,
            winsize=winsize,
            play_flow=False,
            resize_fact_flow=resize_fact_flow)

        plt.figure()
        plt.plot(correlations_orig)

    # %% compare with original movie
    if play_movie:
        m_orig = cm.load_movie_chain(fnames)
        m_els = cm.load(mc.mmap_file)
        ds_ratio = 0.2
        moviehandle = cm.concatenate([
            m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie,
            m_els.resize(1, 1, ds_ratio)
        ],
                                     axis=2)
        moviehandle.play(fr=60, q_max=99.5, magnification=2)  # press q to exit
        del m_orig
        del m_els

    if plot_extras:
        # plot total template
        plt.figure()
        plt.imshow(mc.total_template_els)
        plt.title('Template after iteration')
        # plot x and y corrections
        plt.figure()
        plt.plot(mc.shifts_rig)
        plt.title('Rigid motion correction xy movement')
        plt.legend(['x shift', 'y shift'])
        plt.xlabel('frames')

    # %% MEMORY MAPPING
    border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
    # you can include the boundaries of the FOV if you used the 'copy' option
    # during motion correction, although be careful about the components near
    # the boundaries

    # memory map the file in order 'C'
    fname_new = cm.save_memmap(mc.mmap_file,
                               base_name='memmap_',
                               order='C',
                               border_to_0=border_to_0)  # exclude borders

    # now load the file
    Yr, dims, T = cm.load_memmap(fname_new)
    images = np.reshape(Yr.T, [T] + list(dims), order='F')
    # load frames in python format (T x X x Y)

    # %% restart cluster to clean up memory
    cm.stop_server(dview=dview)
    c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
                                                     n_processes=None,
                                                     single_thread=False)

    # %%  parameters for source extraction and deconvolution

    p = 2  # order of the autoregressive system
    gnb = 2  # number of global background components
    merge_thr = 0.85  # merging threshold, max correlation allowed
    rf = 15  # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
    stride_cnmf = 6  # amount of overlap between the patches in pixels
    K = 2  # number of components per patch
    gSig = [15, 15]  # expected half size of neurons in pixels
    # initialization method (if analyzing dendritic data using 'sparse_nmf')
    method_init = 'greedy_roi'
    ssub = 1  # spatial subsampling during initialization
    tsub = 1  # temporal subsampling during intialization

    # parameters for component evaluation
    opts_dict = {
        'fnames': fnames,
        'fr': fr,
        'nb': gnb,
        'rf': rf,
        'K': K,
        'gSig': gSig,
        'stride': stride_cnmf,
        'method_init': method_init,
        'rolling_sum': True,
        'merge_thr': merge_thr,
        'n_processes': n_processes,
        'only_init': True,
        'ssub': ssub,
        'tsub': tsub
    }

    opts.change_params(params_dict=opts_dict)
    # %% RUN CNMF ON PATCHES
    # First extract spatial and temporal components on patches and combine them
    # for this step deconvolution is turned off (p=0)

    opts.change_params({'p': 0})
    cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
    cnm = cnm.fit(images)

    if plot_extras_cell:
        num_cell_plot = 51
        plt.figure()
        plt.plot(cnm.estimates.C[num_cell_plot, :])
        plt.title('Temporal component')
        plt.legend(['Cell ' + str(num_cell_plot)])
        # plot component sptial profile A
        # first convert back to dense components
        plot_spat_A = cnm.estimates.A[:, num_cell_plot].toarray().reshape(
            list(dims))
        plt.figure()
        plt.imshow(plot_spat_A)
        plt.title('Spatial component cell ' + str(num_cell_plot))

    # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
    #   you can also perform the motion correction plus cnmf fitting steps
    #   simultaneously after defining your parameters object using
    #  cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
    #  cnm1.fit_file(motion_correct=True)

    # %% plot contours of found components
    Cn = cm.local_correlations(images, swap_dim=False)
    Cn[np.isnan(Cn)] = 0
    cnm.estimates.plot_contours(img=Cn)
    plt.title('Contour plots of found components')

    if plot_extras:
        plt.figure()
        plt.imshow(Cn)
        plt.title('Local correlations')

    # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
    cnm.params.change_params({'p': p})
    cnm2 = cnm.refit(images, dview=dview)
    # %% COMPONENT EVALUATION
    # the components are evaluated in three ways:
    #   a) the shape of each component must be correlated with the data
    #   b) a minimum peak SNR is required over the length of a transient
    #   c) each shape passes a CNN based classifier
    min_SNR = 2  # signal to noise ratio for accepting a component
    rval_thr = 0.90  # space correlation threshold for accepting a component
    cnn_thr = 0.99  # threshold for CNN based classifier
    cnn_lowest = 0.1  # neurons with cnn probability lower than this value are rejected

    cnm2.params.set(
        'quality', {
            'decay_time': decay_time,
            'min_SNR': min_SNR,
            'rval_thr': rval_thr,
            'use_cnn': True,
            'min_cnn_thr': cnn_thr,
            'cnn_lowest': cnn_lowest
        })
    cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)

    # %% PLOT COMPONENTS
    cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
    plt.suptitle('Component selection: min_SNR=' + str(min_SNR) +
                 '; rval_thr=' + str(rval_thr) + '; cnn prob range=[' +
                 str(cnn_lowest) + ' ' + str(cnn_thr) + ']')

    # %% VIEW TRACES (accepted and rejected)

    if plot_extras:
        cnm2.estimates.view_components(images,
                                       img=Cn,
                                       idx=cnm2.estimates.idx_components)
        plt.suptitle('Accepted')
        cnm2.estimates.view_components(images,
                                       img=Cn,
                                       idx=cnm2.estimates.idx_components_bad)
        plt.suptitle('Rejected')

    #plt.figure();
    #plt.plot(cnm2.estimates.YrA[0,:]+cnm2.estimates.C[0,:])
    #
    #
    #
    #
    #plt.figure();
    #plt.plot(cnm2.estimates.R[0,:]-cnm2.estimates.YrA[0,:]);
    #plt.plot();
    #plt.show();
    #
    #
    #plt.figure();
    #plt.plot(cnm2.estimates.detrend_df_f[1,:])

    # these store the good and bad components, and next step sorts them
    # cnm2.estimates.idx_components
    # cnm2.estimates.idx_components_bad

    #%% update object with selected components
    #cnm2.estimates.select_components(use_object=True)
    #%% Extract DF/F values
    cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)

    #%% Show final traces

    cnm2.estimates.view_components(img=Cn)
    plt.suptitle("Final results")

    #%% Save the mc data as in cmn struct as well

    ##
    #mc_out = dict(
    #            pw_rigid            = mc.pw_rigid,
    #            fname               = mc.fname,
    #            mmap_file           = mc.mmap_file,
    #            total_template_els  = mc.total_template_els,
    #            total_template_rig  = mc.total_template_rig,
    #            border_nan          = mc.border_nan,
    #            border_to_0         = mc.border_to_0,
    #            x_shifts_els        = mc.x_shifts_els,
    #            y_shifts_els        = mc.y_shifts_els,
    #            Cn                  = Cn
    #            )
    #
    #
    #deepdish.io.save(fnames[0] + '_mc_data.hdf5', mc_out)

    #%% reconstruct denoised movie (press q to exit)
    if play_movie:
        cnm2.estimates.play_movie(images,
                                  q_max=99.9,
                                  gain_res=2,
                                  magnification=2,
                                  bpx=border_to_0,
                                  include_bck=False)  # background not shown

    #%% STOP CLUSTER and clean up log files
    cm.stop_server(dview=dview)
    log_files = glob.glob('*_LOG_*')
    for log_file in log_files:
        os.remove(log_file)

    save_results = True
    if save_results:
        cnm2.save(fnames[0][:-4] + '_results.hdf5')
Example #25
0
def run_gui(path=None, data=None):
    """
    Loads PCF or CNMF data object from a provided path and loads the CaImAn GUI for component inspection.
    This GUI was tweaked to not require any storage-intensive mmap files, but can therefore not show individual frames.
    :param path: optional str, directory of the PCF or CNMF object from which component data should be loaded. If None
                and data=None, a window prompt will open to select a directory where to look for a CNMF/PCF file.
    :param data: optional, data in form of an already loaded cnm object can be provided directly
    :return:
    """

    try:
        cv2.setNumThreads(1)
    except:
        print('Open CV is naturally single threaded')

    try:
        if __IPYTHON__:
            # print(1)
            # this is used for debugging purposes only. allows to reload classes
            # when changed
            get_ipython().magic('load_ext autoreload')
            get_ipython().magic('autoreload 2')
    except NameError:
        print('Not launched under iPython')

    def make_color_img(img, gain=255, min_max=None, out_type=np.uint8):
        if min_max is None:
            min_ = img.min()
            max_ = img.max()
        else:
            min_, max_ = min_max

        img = (img - min_) / (max_ - min_) * gain
        img = img.astype(out_type)
        img = np.dstack([img] * 3)
        return img

    ### FIND DATA ###
    if data is None:  # different conditions on file loading (not necessary if data was already provided)
        if path is None:  # if no path has been given, open a window prompt to select a directory
            F = FileDialog()

            # load object saved by CNMF
            path = F.getExistingDirectory(
                caption='Select folder from which to load a PCF or CNMF file')

        try:  # first try to get CNMF data from a PCF object (should be most up-to-date)
            cnm_obj = pipe.load_pcf(path).cnmf
        except FileNotFoundError:
            try:
                cnm_obj = pipe.load_cnmf(path)
            except FileNotFoundError:
                raise FileNotFoundError(
                    f'Could not find data to load in {path}!')
    else:
        cnm_obj = data

    # movie NOT NEEDED IN VERSION WITHOUT MMAP FILE
    # if not os.path.exists(cnm_obj.mmap_file):
    #     M = FileDialog()
    #     cnm_obj.mmap_file = M.getOpenFileName(caption='Load memory mapped file', filter='*.mmap')[0]
    #
    # if fpath[-3:] == 'nwb':
    #     mov = cm.load(cnm_obj.mmap_file, var_name_hdf5='acquisition/TwoPhotonSeries')
    # else:
    #     mov = cm.load(cnm_obj.mmap_file)

    estimates = cnm_obj.estimates
    params_obj = cnm_obj.params

    # min_mov = np.min(mov)
    # max_mov = np.max(mov)

    if not hasattr(estimates, 'Cn'):
        if not os.path.exists(cnm_obj.mmap_file):
            M = FileDialog()
            cnm_obj.mmap_file = M.getOpenFileName(
                caption='Load memory mapped file', filter='*.mmap')[0]
        mov = cm.load(cnm_obj.mmap_file)

        estimates.Cn = cm.local_correlations(mov, swap_dim=False)
    Cn = estimates.Cn

    # min_mov_denoise = np.min(estimates.A)*estimates.C.min()
    # max_mov_denoise = np.max(estimates.A)*estimates.C.max()
    background_num = -1
    neuron_selected = False
    nr_index = 0

    min_background = np.min(estimates.b, axis=0) * np.min(estimates.f, axis=1)
    max_background = np.max(estimates.b, axis=0) * np.max(estimates.f, axis=1)

    if not hasattr(estimates, 'accepted_list'):
        # if estimates.discarded_components.A.shape[-1] > 0:
        #     estimates.restore_discarded_components()
        estimates.accepted_list = np.array([], dtype=np.int)
        estimates.rejected_list = np.array([], dtype=np.int)
        estimates.img_components = estimates.A.toarray().reshape(
            (estimates.dims[0], estimates.dims[1], -1),
            order='F').transpose([2, 0, 1])
        estimates.cms = np.array([
            scipy.ndimage.measurements.center_of_mass(comp)
            for comp in estimates.img_components
        ])
        estimates.idx_components = np.arange(estimates.nr)
        estimates.idx_components_bad = np.array([])
        estimates.background_image = make_color_img(estimates.Cn)
        # Generate image data
        estimates.img_components /= estimates.img_components.max(
            axis=(1, 2))[:, None, None]
        estimates.img_components *= 255
        estimates.img_components = estimates.img_components.astype(np.uint8)

    def draw_contours_overall(md):
        if md is "reset":
            draw_contours()
        elif md is "neurons":
            if neuron_selected is True:
                #if a specific neuron has been selected, only one contour should be changed while thrshcomp_line is changing
                if nr_index is 0:
                    #if user does not start to move through the frames
                    draw_contours_update(estimates.background_image, img)
                    draw_contours_update(comp2_scaled, img2)
                else:
                    # NEVER CALLED IN THIS VERSION WITHOUT MMAP SINCE NR_INDEX NEVER CHANGES (NO NR_VLINE)
                    draw_contours_update(raw_mov_scaled, img)
                    draw_contours_update(frame_denoise_scaled, img2)
            else:
                #if no specific neuron has been selected, all the contours are changing
                draw_contours()
        else:
            #md is "background":
            return

    def draw_contours():
        global thrshcomp_line, estimates, img
        bkgr_contours = estimates.background_image.copy()

        if len(estimates.idx_components) > 0:
            contours = [
                cv2.findContours(
                    cv2.threshold(img, np.int(thrshcomp_line.value()), 255,
                                  0)[1], cv2.RETR_TREE,
                    cv2.CHAIN_APPROX_SIMPLE)[0]
                for img in estimates.img_components[estimates.idx_components]
            ]
            SNRs = np.array(estimates.r_values)
            iidd = np.array(estimates.idx_components)

            idx1 = np.where(SNRs[iidd] < 0.1)[0]
            idx2 = np.where((SNRs[iidd] >= 0.1) & (SNRs[iidd] < 0.25))[0]
            idx3 = np.where((SNRs[iidd] >= 0.25) & (SNRs[iidd] < 0.5))[0]
            idx4 = np.where((SNRs[iidd] >= 0.5) & (SNRs[iidd] < 0.75))[0]
            idx5 = np.where((SNRs[iidd] >= 0.75) & (SNRs[iidd] < 0.9))[0]
            idx6 = np.where(SNRs[iidd] >= 0.9)[0]

            cv2.drawContours(bkgr_contours,
                             sum([contours[jj] for jj in idx1], []), -1,
                             (255, 0, 0), 1)
            cv2.drawContours(bkgr_contours,
                             sum([contours[jj] for jj in idx2], []), -1,
                             (0, 255, 0), 1)
            cv2.drawContours(bkgr_contours,
                             sum([contours[jj] for jj in idx3], []), -1,
                             (0, 0, 255), 1)
            cv2.drawContours(bkgr_contours,
                             sum([contours[jj] for jj in idx4], []), -1,
                             (255, 255, 0), 1)
            cv2.drawContours(bkgr_contours,
                             sum([contours[jj] for jj in idx5], []), -1,
                             (255, 0, 255), 1)
            cv2.drawContours(bkgr_contours,
                             sum([contours[jj] for jj in idx6], []), -1,
                             (0, 255, 255), 1)

        img.setImage(bkgr_contours, autoLevels=False)

    # pg.setConfigOptions(imageAxisOrder='row-major')

    def draw_contours_update(cf, im):
        global thrshcomp_line, estimates
        curFrame = cf.copy()

        if len(estimates.idx_components) > 0:
            contours = [
                cv2.findContours(
                    cv2.threshold(img, np.int(thrshcomp_line.value()), 255,
                                  0)[1], cv2.RETR_TREE,
                    cv2.CHAIN_APPROX_SIMPLE)[0]
                for img in estimates.img_components[estimates.idx_components]
            ]
            SNRs = np.array(estimates.r_values)
            iidd = np.array(estimates.idx_components)

            idx1 = np.where(SNRs[iidd] < 0.1)[0]
            idx2 = np.where((SNRs[iidd] >= 0.1) & (SNRs[iidd] < 0.25))[0]
            idx3 = np.where((SNRs[iidd] >= 0.25) & (SNRs[iidd] < 0.5))[0]
            idx4 = np.where((SNRs[iidd] >= 0.5) & (SNRs[iidd] < 0.75))[0]
            idx5 = np.where((SNRs[iidd] >= 0.75) & (SNRs[iidd] < 0.9))[0]
            idx6 = np.where(SNRs[iidd] >= 0.9)[0]

            if min_dist_comp in idx1:
                cv2.drawContours(curFrame, contours[min_dist_comp], -1,
                                 (255, 0, 0), 1)
            if min_dist_comp in idx2:
                cv2.drawContours(curFrame, contours[min_dist_comp], -1,
                                 (0, 255, 0), 1)
            if min_dist_comp in idx3:
                cv2.drawContours(curFrame, contours[min_dist_comp], -1,
                                 (0, 0, 255), 1)
            if min_dist_comp in idx4:
                cv2.drawContours(curFrame, contours[min_dist_comp], -1,
                                 (255, 255, 0), 1)
            if min_dist_comp in idx5:
                cv2.drawContours(curFrame, contours[min_dist_comp], -1,
                                 (255, 0, 255), 1)
            if min_dist_comp in idx6:
                cv2.drawContours(curFrame, contours[min_dist_comp], -1,
                                 (0, 255, 255), 1)

        im.setImage(curFrame, autoLevels=False)

#%% START BUILDING THE APPLICATION WINDOW

# Always start by initializing Qt (only once per application)

    app = QtGui.QApplication([])

    # Define a top-level widget to hold everything
    w = QtGui.QWidget()

    # Create some widgets to be placed inside
    btn = QtGui.QPushButton('press me')
    text = QtGui.QLineEdit('enter text')

    # Histogram controller (win)
    win = pg.GraphicsLayoutWidget()
    win.setMaximumWidth(300)
    win.setMinimumWidth(200)
    hist = pg.HistogramLUTItem()  # Contrast/color control
    win.addItem(hist)

    # Plotting windows
    p1 = pg.PlotWidget(
    )  # raw movie window (top-mid), all contours are drawn here
    p2 = pg.PlotWidget(
    )  # trace window (bottom-mid), calcium trace of the selected component
    p3 = pg.PlotWidget(
    )  # denoised movie window (top-right), only selected contour is drawn here

    # parameter table for online evaluation and mode change
    t = ParameterTree()

    # parameter table for neuron selection/sorting
    t_action = ParameterTree()
    action_layout = QtGui.QGridLayout()

    ## Create a grid layout to manage the widgets size and position
    layout = QtGui.QGridLayout()
    w.setLayout(layout)

    # A plot area (ViewBox + axes) for displaying the image
    #p1 = win.addPlot(title="Image here")
    # Item for displaying image data
    img = pg.ImageItem()
    p1.addItem(img)

    img2 = pg.ImageItem()
    p3.addItem(img2)

    hist.setImageItem(img)

    # Draggable line for setting isocurve level
    thrshcomp_line = pg.InfiniteLine(angle=0, movable=True, pen='g')
    hist.vb.addItem(thrshcomp_line)
    hist.vb.setMouseEnabled(y=False)  # makes user interaction a little easier
    thrshcomp_line.setValue(100)
    thrshcomp_line.setZValue(1000)  # bring iso line above contrast controls

    ## Add widgets to the layout in their proper positions
    layout.addWidget(win, 1, 0)  # histogram
    layout.addWidget(p3, 0, 2)  # denoised movie

    layout.addWidget(t, 0, 0)  # upper-right table
    layout.addWidget(t_action, 1, 2)  # bottom-right table
    layout.addWidget(p1, 0, 1)  # raw movie
    layout.addWidget(p2, 1, 1)  # calcium trace window

    #enable only horizontal zoom for the traces component
    p2.setMouseEnabled(x=True, y=False)
    ## Display the widget as a new window
    w.show()

    ## Start the Qt event loop
    app.exec_()

    draw_contours()

    hist.setLevels(estimates.background_image.min(),
                   estimates.background_image.max())

    # Another plot area for displaying ROI data
    #win.nextRow()
    #p2 = win.addPlot(colspan=2)
    p2.setMaximumHeight(250)
    #win.resize(800, 800)
    #win.show()

    # set position and scale of image
    img.scale(1, 1)
    # img.translate(-50, 0)

    # zoom to fit image
    p1.autoRange()

    mode = "reset"
    p2.setTitle("mode: %s" % (mode))

    thrshcomp_line.sigDragged.connect(lambda: draw_contours_overall(mode))

    def imageHoverEvent(event):
        #Show the position, pixel, and value under the mouse cursor.
        global x, y, i, j, val
        pos = event.pos()
        i, j = pos.y(), pos.x()
        i = int(np.clip(i, 0, estimates.background_image.shape[0] - 1))
        j = int(np.clip(j, 0, estimates.background_image.shape[1] - 1))
        val = estimates.background_image[i, j, 0]
        ppos = img.mapToParent(pos)
        x, y = ppos.x(), ppos.y()

    # Monkey-patch the image to use our custom hover function.
    # This is generally discouraged (you should subclass ImageItem instead),
    # but it works for a very simple use like this.
    img.hoverEvent = imageHoverEvent

    def mouseClickEvent(event):
        global mode
        global x, y, i, j, val

        pos = img.mapFromScene(event.pos())
        x = int(pos.x())
        y = int(pos.y())

        if x < 0 or x > mov.shape[1] or y < 0 or y > mov.shape[2]:
            # if the user click outside of the movie, do nothing and jump out of the function
            return

        i, j = pos.y(), pos.x()
        i = int(np.clip(i, 0, estimates.background_image.shape[0] - 1))
        j = int(np.clip(j, 0, estimates.background_image.shape[1] - 1))
        val = estimates.background_image[i, j, 0]

        if mode is "neurons":
            show_neurons_clicked()

    p1.mousePressEvent = mouseClickEvent

    #A general rule in Qt is that if you override one mouse event handler, you must override all of them.
    def release(event):
        pass

    p1.mouseReleaseEvent = release

    def move(event):
        pass

    p1.mouseMoveEvent = move

    ## PARAMS
    params = [{
        'name': 'min_cnn_thr',
        'type': 'float',
        'value': 0.99,
        'limits': (0, 1),
        'step': 0.01
    }, {
        'name': 'cnn_lowest',
        'type': 'float',
        'value': 0.1,
        'limits': (0, 1),
        'step': 0.01
    }, {
        'name': 'rval_thr',
        'type': 'float',
        'value': 0.85,
        'limits': (-1, 1),
        'step': 0.01
    }, {
        'name': 'rval_lowest',
        'type': 'float',
        'value': -1,
        'limits': (-1, 1),
        'step': 0.01
    }, {
        'name': 'min_SNR',
        'type': 'float',
        'value': 2,
        'limits': (0, 20),
        'step': 0.1
    }, {
        'name': 'SNR_lowest',
        'type': 'float',
        'value': 0,
        'limits': (0, 20),
        'step': 0.1
    }, {
        'name': 'RESET',
        'type': 'action'
    }, {
        'name': 'SHOW BACKGROUND',
        'type': 'action'
    }, {
        'name': 'SHOW NEURONS',
        'type': 'action'
    }]

    ## Create tree of Parameter objects
    pars = Parameter.create(name='params', type='group', children=params)

    params_action = [{
        'name': 'Filter components',
        'type': 'bool',
        'value': True,
        'tip': "Filter components"
    }, {
        'name': 'View components',
        'type': 'list',
        'values': ['All', 'Accepted', 'Rejected', 'Unassigned'],
        'value': 'All'
    }, {
        'name': 'ADD GROUP',
        'type': 'action'
    }, {
        'name': 'REMOVE GROUP',
        'type': 'action'
    }, {
        'name': 'ADD SINGLE',
        'type': 'action'
    }, {
        'name': 'REMOVE SINGLE',
        'type': 'action'
    }, {
        'name': 'SAVE OBJECT',
        'type': 'action'
    }]

    pars_action = Parameter.create(name='params_action',
                                   type='group',
                                   children=params_action)

    t_action.setParameters(pars_action, showTop=False)
    t_action.setWindowTitle('Parameter Action')

    def reset_button():
        global mode
        mode = "reset"
        p2.setTitle("mode: %s" % (mode))
        #clear the upper right image
        zeros = np.asarray([[0] * 80 for _ in range(60)])
        img2.setImage(make_color_img(zeros), autoLevels=False)
        draw_contours()

    pars.param('RESET').sigActivated.connect(reset_button)

    def show_background_button():
        global bg_vline, min_background, max_background, background_num
        global mode, background_first_frame_scaled
        #clear thhe upper right image
        zeros = np.asarray([[0] * 80 for _ in range(60)])
        img2.setImage(make_color_img(zeros), autoLevels=False)

        background_num = (background_num + 1) % estimates.f.shape[0]
        mode = "background"
        p2.setTitle("mode: %s %d" % (mode, background_num))

        # display the first frame of the background
        background_first_frame = estimates.b[:, background_num].reshape(
            estimates.dims, order='F')
        min_background_first_frame = np.min(background_first_frame)
        max_background_first_frame = np.max(background_first_frame)
        background_first_frame_scaled = make_color_img(
            background_first_frame,
            min_max=(min_background_first_frame, max_background_first_frame))
        img.setImage(background_first_frame_scaled, autoLevels=False)

        # draw the trace and the infinite line
        trace_background = estimates.f[background_num]
        p2.plot(trace_background, clear=True)
        bg_vline = pg.InfiniteLine(angle=90, movable=True)
        p2.addItem(bg_vline, ignoreBounds=True)
        bg_vline.setValue(0)
        bg_vline.sigPositionChanged.connect(show_background_update)

    def show_background_update():
        global bg_index, min_background, max_background, background_scaled
        bg_index = int(bg_vline.value())
        if bg_index > -1 and bg_index < estimates.f.shape[-1]:
            # upper left component scrolls through the frames of the background
            background = estimates.b[:, background_num].dot(
                estimates.f[background_num, bg_index]).reshape(estimates.dims,
                                                               order='F')
            background_scaled = make_color_img(
                background,
                min_max=(min_background[background_num],
                         max_background[background_num]))
            img.setImage(background_scaled, autoLevels=False)

    pars.param('SHOW BACKGROUND').sigActivated.connect(show_background_button)

    def show_neurons_button():
        global mode, neuron_selected
        mode = "neurons"
        neuron_selected = False
        p2.setTitle("mode: %s" % (mode))
        #clear the upper right image
        zeros = np.asarray([[0] * 80 for _ in range(60)])
        img2.setImage(make_color_img(zeros), autoLevels=False)

    def show_neurons_clicked():
        global nr_index
        global x, y, i, j, val, min_dist_comp, contour_single, neuron_selected, comp2_scaled
        neuron_selected = True
        distances = np.sum(
            ((x, y) - estimates.cms[estimates.idx_components])**2, axis=1)**0.5
        min_dist_comp = np.argmin(distances)
        contour_all = [
            cv2.threshold(img, np.int(thrshcomp_line.value()), 255, 0)[1]
            for img in estimates.img_components[estimates.idx_components]
        ]
        contour_single = contour_all[min_dist_comp]

        # draw the traces (lower left component)
        estimates.components_to_plot = estimates.idx_components[min_dist_comp]
        p2.plot(estimates.C[estimates.components_to_plot] +
                estimates.YrA[estimates.components_to_plot],
                clear=True)

        # plot img (upper left component)
        img.setImage(estimates.background_image, autoLevels=False)
        draw_contours_update(estimates.background_image, img)
        # plot img2 (upper right component)
        comp2 = np.multiply(estimates.Cn, contour_single > 0)
        comp2_scaled = make_color_img(comp2,
                                      min_max=(np.min(comp2), np.max(comp2)))
        img2.setImage(comp2_scaled, autoLevels=False)
        draw_contours_update(comp2_scaled, img2)
        # set title for the upper two components
        p3.setTitle("pos: (%0.1f, %0.1f)  component: %d  value: %g dist:%f" %
                    (x, y, estimates.components_to_plot, val,
                     distances[min_dist_comp]))
        p1.setTitle("pos: (%0.1f, %0.1f)  component: %d  value: %g dist:%f" %
                    (x, y, estimates.components_to_plot, val,
                     distances[min_dist_comp]))

        # draw the infinite line (INACTIVE IN THIS VERSION WITHOUT MMAP FILES)
        # nr_vline = pg.InfiniteLine(angle=90, movable=True)
        # p2.addItem(nr_vline, ignoreBounds=True)
        # nr_vline.setValue(0)
        # nr_vline.sigPositionChanged.connect(show_neurons_update)
        nr_index = 0

    def show_neurons_update():  # NOT CALLED IN THIS VERSION
        global nr_index, frame_denoise_scaled, estimates, raw_mov_scaled
        global min_mov, max_mov, min_mov_denoise, max_mov_denoise
        if neuron_selected is False:
            return
        nr_index = int(nr_vline.value())
        if nr_index > 0 and nr_index < mov[:, 0, 0].shape[0]:
            # upper left component scrolls through the raw movie
            raw_mov = mov[nr_index, :, :]
            raw_mov_scaled = make_color_img(raw_mov,
                                            min_max=(min_mov, max_mov))
            img.setImage(raw_mov_scaled, autoLevels=False)
            draw_contours_update(raw_mov_scaled, img)
            # upper right component scrolls through the denoised movie
            frame_denoise = estimates.A[:, estimates.idx_components].dot(
                estimates.C[estimates.idx_components,
                            nr_index]).reshape(estimates.dims, order='F')
            frame_denoise_scaled = make_color_img(frame_denoise,
                                                  min_max=(min_mov_denoise,
                                                           max_mov_denoise))
            img2.setImage(frame_denoise_scaled, autoLevels=False)
            draw_contours_update(frame_denoise_scaled, img2)

    pars.param('SHOW NEURONS').sigActivated.connect(show_neurons_button)

    def add_group():
        estimates.accepted_list = np.union1d(estimates.accepted_list,
                                             estimates.idx_components)
        estimates.rejected_list = np.setdiff1d(estimates.rejected_list,
                                               estimates.idx_components)
        change(None, None)

    pars_action.param('ADD GROUP').sigActivated.connect(add_group)

    def remove_group():
        estimates.rejected_list = np.union1d(estimates.rejected_list,
                                             estimates.idx_components)
        estimates.accepted_list = np.setdiff1d(estimates.accepted_list,
                                               estimates.idx_components)
        change(None, None)

    pars_action.param('REMOVE GROUP').sigActivated.connect(remove_group)

    def add_single():
        estimates.accepted_list = np.union1d(estimates.accepted_list,
                                             estimates.components_to_plot)
        estimates.rejected_list = np.setdiff1d(estimates.rejected_list,
                                               estimates.components_to_plot)
        change(None, None)

    pars_action.param('ADD SINGLE').sigActivated.connect(add_single)

    def remove_single():
        estimates.rejected_list = np.union1d(estimates.rejected_list,
                                             estimates.components_to_plot)
        estimates.accepted_list = np.setdiff1d(estimates.accepted_list,
                                               estimates.components_to_plot)
        change(None, None)

    pars_action.param('REMOVE SINGLE').sigActivated.connect(remove_single)

    def save_object():
        print('Saving')

        ffll = F.getSaveFileName(filter='*.hdf5')
        print(ffll[0])
        cnm_obj.estimates = estimates
        cnm_obj.save(ffll[0])

    pars_action.param('SAVE OBJECT').sigActivated.connect(save_object)

    def action_pars_activated(param, changes):
        change(None, None)

    pars_action.sigTreeStateChanged.connect(action_pars_activated)

    ## If anything changes in the tree, print a message
    def change(param, changes):
        global estimates, pars, pars_action
        set_par = pars.getValues()
        if pars_action.param('Filter components').value():
            for keyy in set_par.keys():
                params_obj.quality.update({keyy: set_par[keyy][0]})
        else:
            params_obj.quality.update({
                'cnn_lowest': .1,
                'min_cnn_thr': 0.99,
                'rval_thr': 0.85,
                'rval_lowest': -1,
                'min_SNR': 2,
                'SNR_lowest': 0
            })
        estimates.filter_components(
            mov,
            params_obj,
            dview=None,
            select_mode=pars_action.param('View components').value())
        if mode is "background":
            return
        else:
            draw_contours()

    pars.sigTreeStateChanged.connect(change)

    change(None, None)  # set params to default
    t.setParameters(pars, showTop=False)
    t.setWindowTitle('Parameter Quality')

    ## END PARAMS

    ## Display the widget as a new window
    w.show()

    ## Start the Qt event loop
    app.exit(app.exec_())
Example #26
0
def main():
    pass  # For compatibility between running under Spyder and the CLI

    #%% Select file(s) to be processed (download if not present)
    fnames = ['Sue_2x_3000_40_-46.tif']  # filename to be processed
    if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
        fnames = [download_demo(fnames[0])]

#%% First setup some parameters for data and motion correction

# dataset dependent parameters
    fr = 30  # imaging rate in frames per second
    decay_time = 0.4  # length of a typical transient in seconds
    dxy = (2., 2.)  # spatial resolution in x and y in (um per pixel)
    # note the lower than usual spatial resolution here
    max_shift_um = (12., 12.)  # maximum shift in um
    patch_motion_um = (100., 100.)  # patch size for non-rigid correction in um

    # motion correction parameters
    pw_rigid = True  # flag to select rigid vs pw_rigid motion correction
    # maximum allowed rigid shift in pixels
    max_shifts = [int(a / b) for a, b in zip(max_shift_um, dxy)]
    # start a new patch for pw-rigid motion correction every x pixels
    strides = tuple([int(a / b) for a, b in zip(patch_motion_um, dxy)])
    # overlap between pathes (size of patch in pixels: strides+overlaps)
    overlaps = (24, 24)
    # maximum deviation allowed for patch with respect to rigid shifts
    max_deviation_rigid = 3

    mc_dict = {
        'fnames': fnames,
        'fr': fr,
        'decay_time': decay_time,
        'dxy': dxy,
        'pw_rigid': pw_rigid,
        'max_shifts': max_shifts,
        'strides': strides,
        'overlaps': overlaps,
        'max_deviation_rigid': max_deviation_rigid,
        'border_nan': 'copy'
    }

    opts = params.CNMFParams(params_dict=mc_dict)

    # %% play the movie (optional)
    # playing the movie using opencv. It requires loading the movie in memory.
    # To close the video press q
    display_images = False

    if display_images:
        m_orig = cm.load_movie_chain(fnames)
        ds_ratio = 0.2
        moviehandle = m_orig.resize(1, 1, ds_ratio)
        moviehandle.play(q_max=99.5, fr=60, magnification=2)

# %% start a cluster for parallel processing
    c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
                                                     n_processes=None,
                                                     single_thread=False)

    # %%% MOTION CORRECTION
    # first we create a motion correction object with the specified parameters
    mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
    # note that the file is not loaded in memory

    # %% Run (piecewise-rigid motion) correction using NoRMCorre
    mc.motion_correct(save_movie=True)

    # %% compare with original movie
    if display_images:
        m_orig = cm.load_movie_chain(fnames)
        m_els = cm.load(mc.mmap_file)
        ds_ratio = 0.2
        moviehandle = cm.concatenate([
            m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie,
            m_els.resize(1, 1, ds_ratio)
        ],
                                     axis=2)
        moviehandle.play(fr=60, q_max=99.5, magnification=2)  # press q to exit

# %% MEMORY MAPPING
    border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
    # you can include the boundaries of the FOV if you used the 'copy' option
    # during motion correction, although be careful about the components near
    # the boundaries

    # memory map the file in order 'C'
    fname_new = cm.save_memmap(mc.mmap_file,
                               base_name='memmap_',
                               order='C',
                               border_to_0=border_to_0)  # exclude borders

    # now load the file
    Yr, dims, T = cm.load_memmap(fname_new)
    images = np.reshape(Yr.T, [T] + list(dims), order='F')
    # load frames in python format (T x X x Y)

    # %% restart cluster to clean up memory
    cm.stop_server(dview=dview)
    c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
                                                     n_processes=None,
                                                     single_thread=False)

    # %%  parameters for source extraction and deconvolution
    p = 1  # order of the autoregressive system
    gnb = 2  # number of global background components
    merge_thr = 0.85  # merging threshold, max correlation allowed
    rf = 15
    # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
    stride_cnmf = 6  # amount of overlap between the patches in pixels
    K = 4  # number of components per patch
    gSig = [4, 4]  # expected half size of neurons in pixels
    # initialization method (if analyzing dendritic data using 'sparse_nmf')
    method_init = 'greedy_roi'
    ssub = 2  # spatial subsampling during initialization
    tsub = 2  # temporal subsampling during intialization

    # parameters for component evaluation
    opts_dict = {
        'fnames': fnames,
        'fr': fr,
        'nb': gnb,
        'rf': rf,
        'K': K,
        'gSig': gSig,
        'stride': stride_cnmf,
        'method_init': method_init,
        'rolling_sum': True,
        'merge_thr': merge_thr,
        'n_processes': n_processes,
        'only_init': True,
        'ssub': ssub,
        'tsub': tsub
    }

    opts.change_params(params_dict=opts_dict)
    # %% RUN CNMF ON PATCHES
    # First extract spatial and temporal components on patches and combine them
    # for this step deconvolution is turned off (p=0)

    opts.change_params({'p': 0})
    cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
    cnm = cnm.fit(images)

    # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
    #   you can also perform the motion correction plus cnmf fitting steps
    #   simultaneously after defining your parameters object using
    #  cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
    #  cnm1.fit_file(motion_correct=True)

    # %% plot contours of found components
    Cn = cm.local_correlations(images, swap_dim=False)
    Cn[np.isnan(Cn)] = 0
    cnm.estimates.plot_contours(img=Cn)
    plt.title('Contour plots of found components')

    # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
    cnm.params.change_params({'p': p})
    cnm2 = cnm.refit(images, dview=dview)
    # %% COMPONENT EVALUATION
    # the components are evaluated in three ways:
    #   a) the shape of each component must be correlated with the data
    #   b) a minimum peak SNR is required over the length of a transient
    #   c) each shape passes a CNN based classifier
    min_SNR = 2  # signal to noise ratio for accepting a component
    rval_thr = 0.85  # space correlation threshold for accepting a component
    cnn_thr = 0.99  # threshold for CNN based classifier
    cnn_lowest = 0.1  # neurons with cnn probability lower than this value are rejected

    cnm2.params.set(
        'quality', {
            'decay_time': decay_time,
            'min_SNR': min_SNR,
            'rval_thr': rval_thr,
            'use_cnn': True,
            'min_cnn_thr': cnn_thr,
            'cnn_lowest': cnn_lowest
        })
    cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
    # %% PLOT COMPONENTS
    cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)

    # %% VIEW TRACES (accepted and rejected)

    if display_images:
        cnm2.estimates.view_components(images,
                                       img=Cn,
                                       idx=cnm2.estimates.idx_components)
        cnm2.estimates.view_components(images,
                                       img=Cn,
                                       idx=cnm2.estimates.idx_components_bad)
    #%% update object with selected components
    cnm2.estimates.select_components(use_object=True)
    #%% Extract DF/F values
    cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)

    #%% Show final traces
    cnm2.estimates.view_components(img=Cn)

    #%% reconstruct denoised movie (press q to exit)
    if display_images:
        cnm2.estimates.play_movie(images,
                                  q_max=99.9,
                                  gain_res=2,
                                  magnification=2,
                                  bpx=border_to_0,
                                  include_bck=False)  # background not shown

    #%% STOP CLUSTER and clean up log files
    cm.stop_server(dview=dview)
    log_files = glob.glob('*_LOG_*')
    for log_file in log_files:
        os.remove(log_file)
Example #27
0
def test_general():
    """  General Test of pipeline with comparison against ground truth
    A shorter version than the demo pipeline that calls comparison for the real test work



        Raises:
      ---------
        params_movie

        params_cnmf

        rig correction

        cnmf on patch

        cnmf full frame

        not able to read the file

        no groundtruth


    """
#\bug
#\warning

    global params_movie
    global params_diplay
    fname = params_movie['fname']
    niter_rig = params_movie['niter_rig']
    max_shifts = params_movie['max_shifts']
    splits_rig = params_movie['splits_rig']
    num_splits_to_process_rig = params_movie['num_splits_to_process_rig']

    cwd = os.getcwd()
    fname = download_demo(fname[0])
    m_orig = cm.load(fname)
    min_mov = m_orig[:400].min()
    comp = comparison.Comparison()
    comp.dims = np.shape(m_orig)[1:]


################ RIG CORRECTION #################
    t1 = time.time()
    mc = MotionCorrect(fname, min_mov,
                       max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig,
                       num_splits_to_process_rig=num_splits_to_process_rig,
                       shifts_opencv=True, nonneg_movie=True)
    mc.motion_correct_rigid(save_movie=True)
    m_rig = cm.load(mc.fname_tot_rig)
    bord_px_rig = np.ceil(np.max(mc.shifts_rig)).astype(np.int)
    comp.comparison['rig_shifts']['timer'] = time.time() - t1
    comp.comparison['rig_shifts']['ourdata'] = mc.shifts_rig
###########################################

    if 'max_shifts' not in params_movie:
        fnames = params_movie['fname']
        border_to_0 = 0
    else:  # elif not params_movie.has_key('overlaps'):
        fnames = mc.fname_tot_rig
        border_to_0 = bord_px_rig
        m_els = m_rig

    idx_xy = None
    add_to_movie = -np.nanmin(m_els) + 1  # movie must be positive
    remove_init = 0
    downsample_factor = 1
    base_name = fname[0].split('/')[-1][:-4]
    name_new = cm.save_memmap_each(fnames, base_name=base_name, resize_fact=(
        1, 1, downsample_factor), remove_init=remove_init,
        idx_xy=idx_xy, add_to_movie=add_to_movie, border_to_0=border_to_0)
    name_new.sort()

    if len(name_new) > 1:
        fname_new = cm.save_memmap_join(
            name_new, base_name='Yr', n_chunks=params_movie['n_chunks'], dview=None)
    else:
        print('One file only, not saving!')
        fname_new = name_new[0]

    Yr, dims, T = cm.load_memmap(fname_new)
    images = np.reshape(Yr.T, [T] + list(dims), order='F')
    Y = np.reshape(Yr, dims + (T,), order='F')

    if np.min(images) < 0:
        # TODO: should do this in an automatic fashion with a while loop at the 367 line
        raise Exception('Movie too negative, add_to_movie should be larger')
    if np.sum(np.isnan(images)) > 0:
        # TODO: same here
        raise Exception(
            'Movie contains nan! You did not remove enough borders')

    Cn = cm.local_correlations(Y)
    Cn[np.isnan(Cn)] = 0
    p = params_movie['p']
    merge_thresh = params_movie['merge_thresh']
    rf = params_movie['rf']
    stride_cnmf = params_movie['stride_cnmf']
    K = params_movie['K']
    init_method = params_movie['init_method']
    gSig = params_movie['gSig']
    alpha_snmf = params_movie['alpha_snmf']

    if params_movie['is_dendrites'] == True:
        if params_movie['init_method'] is not 'sparse_nmf':
            raise Exception('dendritic requires sparse_nmf')
        if params_movie['alpha_snmf'] is None:
            raise Exception('need to set a value for alpha_snmf')


################ CNMF PART PATCH #################
    t1 = time.time()
    cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=params_movie['merge_thresh'], p=params_movie['p'],
                    dview=None, rf=rf, stride=stride_cnmf, memory_fact=params_movie['memory_fact'],
                    method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=params_movie[
                        'only_init_patch'],
                    gnb=params_movie['gnb'], method_deconvolution='oasis')
    comp.cnmpatch = copy.copy(cnm)
    cnm = cnm.fit(images)
    A_tot = cnm.A
    C_tot = cnm.C
    YrA_tot = cnm.YrA
    b_tot = cnm.b
    f_tot = cnm.f
    # DISCARDING
    print(('Number of components:' + str(A_tot.shape[-1])))
    final_frate = params_movie['final_frate']
    # threshold on space consistency
    r_values_min = params_movie['r_values_min_patch']
    # threshold on time variability
    fitness_min = params_movie['fitness_delta_min_patch']
    fitness_delta_min = params_movie['fitness_delta_min_patch']
    Npeaks = params_movie['Npeaks']
    traces = C_tot + YrA_tot
    idx_components, idx_components_bad = estimate_components_quality(
        traces, Y, A_tot, C_tot, b_tot, f_tot, final_frate=final_frate,
        Npeaks=Npeaks, r_values_min=r_values_min, fitness_min=fitness_min,
        fitness_delta_min=fitness_delta_min)
    #######
    A_tot = A_tot.tocsc()[:, idx_components]
    C_tot = C_tot[idx_components]
    comp.comparison['cnmf_on_patch']['timer'] = time.time() - t1
    comp.comparison['cnmf_on_patch']['ourdata'] = [A_tot.copy(), C_tot.copy()]
#################### ########################


################ CNMF PART FULL #################
    t1 = time.time()
    cnm = cnmf.CNMF(n_processes=1, k=A_tot.shape, gSig=gSig, merge_thresh=merge_thresh, p=p, Ain=A_tot, Cin=C_tot,
                    f_in=f_tot, rf=None, stride=None, method_deconvolution='oasis')
    cnm = cnm.fit(images)
    # DISCARDING
    A, C, b, f, YrA, sn = cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, cnm.sn
    final_frate = params_movie['final_frate']
    # threshold on space consistency
    r_values_min = params_movie['r_values_min_full']
    # threshold on time variability
    fitness_min = params_movie['fitness_delta_min_full']
    fitness_delta_min = params_movie['fitness_delta_min_full']
    Npeaks = params_movie['Npeaks']
    traces = C + YrA
    idx_components, idx_components_bad, fitness_raw, fitness_delta, r_values = estimate_components_quality(
        traces, Y, A, C, b, f, final_frate=final_frate, Npeaks=Npeaks, r_values_min=r_values_min,
        fitness_min=fitness_min,
        fitness_delta_min=fitness_delta_min, return_all=True)
    ##########
    A_tot_full = A_tot.tocsc()[:, idx_components]
    C_tot_full = C_tot[idx_components]
    comp.comparison['cnmf_full_frame']['timer'] = time.time() - t1
    comp.comparison['cnmf_full_frame']['ourdata'] = [
        A_tot_full.copy(), C_tot_full.copy()]
#################### ########################
    comp.save_with_compare(istruth=False, params=params_movie, Cn=Cn)
    log_files = glob.glob('*_LOG_*')
    try:
        for log_file in log_files:
            os.remove(log_file)
    except:
        print('Cannot remove log files')
############ assertions ##################
    pb = False
    if (comp.information['differences']['params_movie']):
        print("you need to set the same movie parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)")
        pb = True
    if (comp.information['differences']['params_cnm']):
        print("you need to set the same cnmf parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)")
        pb = True
    if (comp.information['diff']['rig']['isdifferent']):
        print("the rigid shifts are different from the groundtruth ")
        pb = True
    if (comp.information['diff']['cnmpatch']['isdifferent']):
        print("the cnmf on patch produces different results than the groundtruth ")
        pb = True
    if (comp.information['diff']['cnmfull']['isdifferent']):
        print("the cnmf full frame produces different  results than the groundtruth ")
        pb = True

    assert (not pb)
Example #28
0
def test_general():
    """  General Test of pipeline with comparison against ground truth
    A shorter version than the demo pipeline that calls comparison for the real test work



        Raises:
      ---------
        params_movie

        params_cnmf

        rig correction

        cnmf on patch

        cnmf full frame

        not able to read the file

        no groundtruth


    """
    #\bug
    #\warning

    global params_movie
    global params_diplay
    fname = params_movie['fname']
    niter_rig = params_movie['niter_rig']
    max_shifts = params_movie['max_shifts']
    splits_rig = params_movie['splits_rig']
    num_splits_to_process_rig = params_movie['num_splits_to_process_rig']

    cwd = os.getcwd()
    fname = download_demo(fname[0])
    m_orig = cm.load(fname)
    min_mov = m_orig[:400].min()
    comp = comparison.Comparison()
    comp.dims = np.shape(m_orig)[1:]

    ################ RIG CORRECTION #################
    t1 = time.time()
    mc = MotionCorrect(fname,
                       min_mov,
                       max_shifts=max_shifts,
                       niter_rig=niter_rig,
                       splits_rig=splits_rig,
                       num_splits_to_process_rig=num_splits_to_process_rig,
                       shifts_opencv=True,
                       nonneg_movie=True)
    mc.motion_correct_rigid(save_movie=True)
    m_rig = cm.load(mc.fname_tot_rig)
    bord_px_rig = np.ceil(np.max(mc.shifts_rig)).astype(np.int)
    comp.comparison['rig_shifts']['timer'] = time.time() - t1
    comp.comparison['rig_shifts']['ourdata'] = mc.shifts_rig
    ###########################################

    if 'max_shifts' not in params_movie:
        fnames = params_movie['fname']
        border_to_0 = 0
    else:  # elif not params_movie.has_key('overlaps'):
        fnames = mc.fname_tot_rig
        border_to_0 = bord_px_rig
        m_els = m_rig

    idx_xy = None
    add_to_movie = -np.nanmin(m_els) + 1  # movie must be positive
    remove_init = 0
    downsample_factor = 1
    base_name = fname[0].split('/')[-1][:-4]
    name_new = cm.save_memmap_each(fnames,
                                   base_name=base_name,
                                   resize_fact=(1, 1, downsample_factor),
                                   remove_init=remove_init,
                                   idx_xy=idx_xy,
                                   add_to_movie=add_to_movie,
                                   border_to_0=border_to_0)
    name_new.sort()

    if len(name_new) > 1:
        fname_new = cm.save_memmap_join(name_new,
                                        base_name='Yr',
                                        n_chunks=params_movie['n_chunks'],
                                        dview=None)
    else:
        logging.warning('One file only, not saving!')
        fname_new = name_new[0]

    Yr, dims, T = cm.load_memmap(fname_new)
    images = np.reshape(Yr.T, [T] + list(dims), order='F')
    Y = np.reshape(Yr, dims + (T, ), order='F')

    if np.min(images) < 0:
        # TODO: should do this in an automatic fashion with a while loop at the 367 line
        raise Exception('Movie too negative, add_to_movie should be larger')
    if np.sum(np.isnan(images)) > 0:
        # TODO: same here
        raise Exception(
            'Movie contains nan! You did not remove enough borders')

    Cn = cm.local_correlations(Y)
    Cn[np.isnan(Cn)] = 0
    p = params_movie['p']
    merge_thresh = params_movie['merge_thresh']
    rf = params_movie['rf']
    stride_cnmf = params_movie['stride_cnmf']
    K = params_movie['K']
    init_method = params_movie['init_method']
    gSig = params_movie['gSig']
    alpha_snmf = params_movie['alpha_snmf']

    if params_movie['is_dendrites'] == True:
        if params_movie['init_method'] is not 'sparse_nmf':
            raise Exception('dendritic requires sparse_nmf')
        if params_movie['alpha_snmf'] is None:
            raise Exception('need to set a value for alpha_snmf')

################ CNMF PART PATCH #################
    t1 = time.time()
    cnm = cnmf.CNMF(n_processes=1,
                    k=K,
                    gSig=gSig,
                    merge_thresh=params_movie['merge_thresh'],
                    p=params_movie['p'],
                    dview=None,
                    rf=rf,
                    stride=stride_cnmf,
                    memory_fact=params_movie['memory_fact'],
                    method_init=init_method,
                    alpha_snmf=alpha_snmf,
                    only_init_patch=params_movie['only_init_patch'],
                    gnb=params_movie['gnb'],
                    method_deconvolution='oasis')
    comp.cnmpatch = copy.copy(cnm)
    comp.cnmpatch.estimates = None
    cnm = cnm.fit(images)
    A_tot = cnm.estimates.A
    C_tot = cnm.estimates.C
    YrA_tot = cnm.estimates.YrA
    b_tot = cnm.estimates.b
    f_tot = cnm.estimates.f
    # DISCARDING
    logging.info(('Number of components:' + str(A_tot.shape[-1])))
    final_frate = params_movie['final_frate']
    # threshold on space consistency
    r_values_min = params_movie['r_values_min_patch']
    # threshold on time variability
    fitness_min = params_movie['fitness_delta_min_patch']
    fitness_delta_min = params_movie['fitness_delta_min_patch']
    Npeaks = params_movie['Npeaks']
    traces = C_tot + YrA_tot
    idx_components, idx_components_bad = estimate_components_quality(
        traces,
        Y,
        A_tot,
        C_tot,
        b_tot,
        f_tot,
        final_frate=final_frate,
        Npeaks=Npeaks,
        r_values_min=r_values_min,
        fitness_min=fitness_min,
        fitness_delta_min=fitness_delta_min)
    #######
    A_tot = A_tot.tocsc()[:, idx_components]
    C_tot = C_tot[idx_components]
    comp.comparison['cnmf_on_patch']['timer'] = time.time() - t1
    comp.comparison['cnmf_on_patch']['ourdata'] = [A_tot.copy(), C_tot.copy()]
    #################### ########################

    ################ CNMF PART FULL #################
    t1 = time.time()
    cnm = cnmf.CNMF(n_processes=1,
                    k=A_tot.shape,
                    gSig=gSig,
                    merge_thresh=merge_thresh,
                    p=p,
                    Ain=A_tot,
                    Cin=C_tot,
                    f_in=f_tot,
                    rf=None,
                    stride=None,
                    method_deconvolution='oasis')
    cnm = cnm.fit(images)
    # DISCARDING
    A, C, b, f, YrA, sn = cnm.estimates.A, cnm.estimates.C, cnm.estimates.b, cnm.estimates.f, cnm.estimates.YrA, cnm.estimates.sn
    final_frate = params_movie['final_frate']
    # threshold on space consistency
    r_values_min = params_movie['r_values_min_full']
    # threshold on time variability
    fitness_min = params_movie['fitness_delta_min_full']
    fitness_delta_min = params_movie['fitness_delta_min_full']
    Npeaks = params_movie['Npeaks']
    traces = C + YrA
    idx_components, idx_components_bad, fitness_raw, fitness_delta, r_values = estimate_components_quality(
        traces,
        Y,
        A,
        C,
        b,
        f,
        final_frate=final_frate,
        Npeaks=Npeaks,
        r_values_min=r_values_min,
        fitness_min=fitness_min,
        fitness_delta_min=fitness_delta_min,
        return_all=True)
    ##########
    A_tot_full = A_tot.tocsc()[:, idx_components]
    C_tot_full = C_tot[idx_components]
    comp.comparison['cnmf_full_frame']['timer'] = time.time() - t1
    comp.comparison['cnmf_full_frame']['ourdata'] = [
        A_tot_full.copy(), C_tot_full.copy()
    ]
    #################### ########################
    comp.save_with_compare(istruth=False, params=params_movie, Cn=Cn)
    log_files = glob.glob('*_LOG_*')
    try:
        for log_file in log_files:
            os.remove(log_file)
    except:
        logging.warning('Cannot remove log files')
############ assertions ##################
    pb = False
    if (comp.information['differences']['params_movie']):
        logging.error(
            "you need to set the same movie parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)"
        )
        pb = True
    if (comp.information['differences']['params_cnm']):
        logging.warning(
            "you need to set the same cnmf parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)"
        )
        # pb = True
    if (comp.information['diff']['rig']['isdifferent']):
        logging.error("the rigid shifts are different from the groundtruth ")
        pb = True
    if (comp.information['diff']['cnmpatch']['isdifferent']):
        logging.error(
            "the cnmf on patch produces different results than the groundtruth "
        )
        pb = True
    if (comp.information['diff']['cnmfull']['isdifferent']):
        logging.error(
            "the cnmf full frame produces different  results than the groundtruth "
        )
        pb = True

    assert (not pb)
Example #29
0
                                    dview=dview)
else:
    fname_new = name_new[0]
#%%
from time import time
# fname_new='Yr_d1_501_d2_398_d3_1_order_F_frames_369_.mmap'
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
Y = np.reshape(Yr, dims + (T, ), order='F')
#%%
# if np.min(images)<0:
#    raise Exception('Movie too negative, add_to_movie should be larger')
#%%
if 0:
    Cn = cm.local_correlations(
        Y[:, :, ::np.int(old_div(np.shape(images)[0], 3000))])
    pl.imshow(Cn)

#%%
# if not is_patches:
#    #%%
#    K = 35  # number of neurons expected per patch
#    gSig = [7, 7]  # expected half size of neurons
#    merge_thresh = 0.8  # merging threshold, max correlation allowed
#    p = 2  # order of the autoregressive system
#    cnm = cnmf.CNMF(n_processes, method_init=init_method, k=K, gSig=gSig, merge_thresh=merge_thresh,
#                    p=p, dview=dview, Ain=None,method_deconvolution='oasis')
#    cnm = cnm.fit(images)
#
# %%
# else:
Example #30
0
                                          filter='*.mmap')[0]

if fpath[-3:] == 'nwb':
    mov = cm.load(cnm_obj.mmap_file,
                  var_name_hdf5='acquisition/TwoPhotonSeries')
else:
    mov = cm.load(cnm_obj.mmap_file)

estimates = cnm_obj.estimates
params_obj = cnm_obj.params

min_mov = np.min(mov)
max_mov = np.max(mov)

if not hasattr(estimates, 'Cn'):
    estimates.Cn = cm.local_correlations(mov, swap_dim=False)
Cn = estimates.Cn

min_mov_denoise = np.min(estimates.A) * estimates.C.min()
max_mov_denoise = np.max(estimates.A) * estimates.C.max()
background_num = -1
neuron_selected = False
nr_index = 0

min_background = np.min(estimates.b, axis=0) * np.min(estimates.f, axis=1)
max_background = np.max(estimates.b, axis=0) * np.max(estimates.f, axis=1)

if not hasattr(estimates, 'accepted_list'):
    # if estimates.discarded_components.A.shape[-1] > 0:
    #     estimates.restore_discarded_components()
    estimates.accepted_list = np.array([], dtype=np.int)
Example #31
0
border_to_0 = bord_px_els     # number of pixels to exclude
fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C',
                           border_to_0=border_to_0)  # exclude borders

# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)

#%% restart cluster to clean up memory
dview.terminate()
c, dview, n_processes = cm.cluster.setup_cluster(
    backend='local', n_processes=None, single_thread=False)
#%%
Cn = cm.local_correlations(images.transpose(1, 2, 0))
Cn[np.isnan(Cn)] = 0
plt.figure()
plt.imshow(Cn,vmax = np.percentile(Cn,95))
#%% RUN CNMF ON PATCHES

# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
t1 = time.time()

cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh,
                p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1,
                method_init=init_method, alpha_snmf=alpha_snmf,
                only_init_patch=False, gnb=gnb, border_pix=bord_px_els, nb_patch=2, low_rank_background=False)
cnm = cnm.fit(images)
Example #32
0
def main():
    pass  # For compatibility between running under Spyder and the CLI

    #%% load data

    fname = os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif')
    Y = cm.load(fname).astype(np.float32)  #
    # used as a background image
    Cn = cm.local_correlations(Y.transpose(1, 2, 0))
    #%% set up some parameters

    # frame rate (Hz)
    fr = 10
    # approximate length of transient event in seconds
    decay_time = 0.5
    # expected half size of neurons
    gSig = [6, 6]
    # order of AR indicator dynamics
    p = 1
    # minimum SNR for accepting new components
    min_SNR = 3.5
    # correlation threshold for new component inclusion
    rval_thr = 0.90
    # number of background components
    gnb = 3

    # set up some additional supporting parameters needed for the algorithm (these are default values but change according to dataset characteristics)

    # number of shapes to be updated each time (put this to a finite small value to increase speed)
    max_comp_update_shape = np.inf
    # maximum number of expected components used for memory pre-allocation (exaggerate here)
    expected_comps = 50
    # number of timesteps to consider when testing new neuron candidates
    N_samples = np.ceil(fr * decay_time)
    # exceptionality threshold
    thresh_fitness_raw = log_ndtr(-min_SNR) * N_samples
    # total length of file
    T1 = Y.shape[0]

    # set up CNMF initialization parameters

    # merging threshold, max correlation allowed
    merge_thresh = 0.8
    # number of frames for initialization (presumably from the first file)
    initbatch = 400
    # size of patch
    patch_size = 32
    # amount of overlap between patches
    stride = 3
    # max number of components in each patch
    K = 4

    #%% obtain initial batch file used for initialization
    # memory map file (not needed)
    fname_new = Y[:initbatch].save(os.path.join(caiman_datadir(),
                                                'example_movies', 'demo.mmap'),
                                   order='C')
    Yr, dims, T = cm.load_memmap(fname_new)
    images = np.reshape(Yr.T, [T] + list(dims), order='F')
    Cn_init = cm.local_correlations(np.reshape(Yr, dims + (T, ), order='F'))

    #%% RUN (offline) CNMF algorithm on the initial batch
    pl.close('all')
    cnm_init = cnmf.CNMF(2,
                         k=K,
                         gSig=gSig,
                         merge_thresh=merge_thresh,
                         fr=fr,
                         p=p,
                         rf=patch_size // 2,
                         stride=stride,
                         skip_refinement=False,
                         normalize_init=False,
                         options_local_NMF=None,
                         minibatch_shape=100,
                         minibatch_suff_stat=5,
                         update_num_comps=True,
                         rval_thr=rval_thr,
                         thresh_fitness_delta=-50,
                         gnb=gnb,
                         decay_time=decay_time,
                         thresh_fitness_raw=thresh_fitness_raw,
                         batch_update_suff_stat=False,
                         max_comp_update_shape=max_comp_update_shape,
                         expected_comps=expected_comps,
                         dview=None,
                         min_SNR=min_SNR)

    cnm_init = cnm_init.fit(images)

    print(('Number of components:' + str(cnm_init.estimates.A.shape[-1])))

    pl.figure()
    crd = plot_contours(cnm_init.estimates.A.tocsc(), Cn_init, thr=0.9)

    #%% run (online) OnACID algorithm

    cnm = deepcopy(cnm_init)
    cnm.params.data['dims'] = (60, 80)
    cnm._prepare_object(np.asarray(Yr), T1)

    t = initbatch

    Y_ = cm.load(fname)[initbatch:].astype(np.float32)
    for frame_count, frame in enumerate(Y_):
        cnm.fit_next(t, frame.copy().reshape(-1, order='F'))
        t += 1

#%% extract the results

    C, f = cnm.estimates.C_on[gnb:cnm.M], cnm.estimates.C_on[:gnb]
    A, b = cnm.estimates.Ab[:, gnb:cnm.M], cnm.estimates.Ab[:, :gnb]
    print(('Number of components:' + str(A.shape[-1])))

    #%% pass through the CNN classifier with a low threshold (keeps clearer neuron shapes and excludes processes)
    use_CNN = True
    if use_CNN:
        # threshold for CNN classifier
        thresh_cnn = 0.1
        from caiman.components_evaluation import evaluate_components_CNN
        predictions, final_crops = evaluate_components_CNN(
            A,
            dims,
            gSig,
            model_name=os.path.join(caiman_datadir(), 'model', 'cnn_model'))
        A_exclude, C_exclude = A[:, predictions[:, 1] < thresh_cnn], C[
            predictions[:, 1] < thresh_cnn]
        A, C = A[:,
                 predictions[:,
                             1] >= thresh_cnn], C[predictions[:,
                                                              1] >= thresh_cnn]
        noisyC = cnm.estimates.noisyC[gnb:cnm.M]
        YrA = noisyC[predictions[:, 1] >= thresh_cnn] - C
    else:
        YrA = cnm.estimates.noisyC[gnb:cnm.M] - C

#%% plot results
    pl.figure()
    crd = cm.utils.visualization.plot_contours(A, Cn, thr=0.9)

    view_patches_bar(Yr, A, C, b, f, dims[0], dims[1], YrA, img=Cn)
Example #33
0
def save_mc(mc, caiman_fp, is3D):
    """
    DataJoint Imaging Element - CaImAn Integration
    Run these commands after the CaImAn analysis has completed.
    This will save the relevant motion correction data into the '*.hdf5' file.
    Please do not clear variables from memory prior to running these commands.
    The motion correction (mc) object will be read from memory.

    'mc' :                CaImAn motion correction object
    'caiman_fp' :         CaImAn output (*.hdf5) file path

    'shifts_rig' :        Rigid transformation x and y shifts per frame
    'x_shifts_els' :      Non rigid transformation x shifts per frame per block
    'y_shifts_els' :      Non rigid transformation y shifts per frame per block
    """

    # Load motion corrected mmap image
    mc_image = cm.load(mc.mmap_file, is3D=is3D)

    # Compute motion corrected summary images
    average_image = np.mean(mc_image, axis=0)
    max_image = np.max(mc_image, axis=0)

    # Compute motion corrected correlation image
    correlation_image = cm.local_correlations(
        mc_image.transpose((1, 2, 3, 0) if is3D else (1, 2, 0)))
    correlation_image[np.isnan(correlation_image)] = 0

    # Compute mc.coord_shifts_els
    grid = []
    if is3D:
        for _, _, _, x, y, z, _ in cm.motion_correction.sliding_window_3d(
                mc_image[0, :, :, :], mc.overlaps, mc.strides):
            grid.append([
                x, x + mc.overlaps[0] + mc.strides[0], y,
                y + mc.overlaps[1] + mc.strides[1], z,
                z + mc.overlaps[2] + mc.strides[2]
            ])
    else:
        for _, _, x, y, _ in cm.motion_correction.sliding_window(
                mc_image[0, :, :], mc.overlaps, mc.strides):
            grid.append([
                x, x + mc.overlaps[0] + mc.strides[0], y,
                y + mc.overlaps[1] + mc.strides[1]
            ])

    # Open hdf5 file and create 'motion_correction' group
    h5f = h5py.File(caiman_fp, 'r+')
    h5g = h5f.require_group("motion_correction")

    # Write motion correction shifts and motion corrected summary images to hdf5 file
    if mc.pw_rigid:
        h5g.require_dataset("x_shifts_els",
                            shape=np.shape(mc.x_shifts_els),
                            data=mc.x_shifts_els,
                            dtype=mc.x_shifts_els[0][0].dtype)
        h5g.require_dataset("y_shifts_els",
                            shape=np.shape(mc.y_shifts_els),
                            data=mc.y_shifts_els,
                            dtype=mc.y_shifts_els[0][0].dtype)
        if is3D:
            h5g.require_dataset("z_shifts_els",
                                shape=np.shape(mc.z_shifts_els),
                                data=mc.z_shifts_els,
                                dtype=mc.z_shifts_els[0][0].dtype)

        h5g.require_dataset("coord_shifts_els",
                            shape=np.shape(grid),
                            data=grid,
                            dtype=type(grid[0][0]))

        # For CaImAn, reference image is still a 2D array even for the case of 3D
        # Assume that the same ref image is used for all the planes
        reference_image = np.tile(mc.total_template_els, (1, 1, correlation_image.shape[-1]))\
            if is3D else mc.total_template_els
    else:
        h5g.require_dataset("shifts_rig",
                            shape=np.shape(mc.shifts_rig),
                            data=mc.shifts_rig,
                            dtype=mc.shifts_rig[0].dtype)
        h5g.require_dataset("coord_shifts_rig",
                            shape=np.shape(grid),
                            data=grid,
                            dtype=type(grid[0][0]))
        reference_image = np.tile(mc.total_template_rig, (1, 1, correlation_image.shape[-1]))\
            if is3D else mc.total_template_rig

    h5g.require_dataset("reference_image",
                        shape=np.shape(reference_image),
                        data=reference_image,
                        dtype=reference_image.dtype)
    h5g.require_dataset("correlation_image",
                        shape=np.shape(correlation_image),
                        data=correlation_image,
                        dtype=correlation_image.dtype)
    h5g.require_dataset("average_image",
                        shape=np.shape(average_image),
                        data=average_image,
                        dtype=average_image.dtype)
    h5g.require_dataset("max_image",
                        shape=np.shape(max_image),
                        data=max_image,
                        dtype=max_image.dtype)

    # Close hdf5 file
    h5f.close()