def _processFrame(self, frame, frame_number): ''' Do some basic processing on a single frame Raises NaNFrameException if a frame contains NaN Returns the normalized/etc modified frame ''' t = time.time() if frame is None: raise ObjectNotFoundError if np.isnan(np.sum(frame)): raise NaNFrameException frame = frame.astype( np.float32) #or require float32 from image acquistion if self.onAc.params.get('online', 'ds_factor') > 1: frame = cv2.resize(frame, self.onAc.img_norm.shape[::-1]) # TODO check for params, onAc componenets before calling, or except if self.onAc.params.get('online', 'normalize'): frame -= self.onAc.img_min if self.onAc.params.get('online', 'motion_correct'): try: templ = self.onAc.estimates.Ab.dot( self.onAc.estimates.C_on[:self.onAc.M, (frame_number - 1)] ).reshape( # self.onAc.estimates.C_on[:self.onAc.M, (frame_number-1)%self.onAc.window]).reshape( self.onAc.params.get('data', 'dims'), order='F') * self.onAc.img_norm except Exception as e: logger.error('Unknown exception {0}'.format(e)) raise Exception if self.onAc.params.get('motion', 'pw_rigid'): frame_cor, shift = tile_and_correct( frame, templ, self.onAc.params.motion['strides'], self.onAc.params.motion['overlaps'], self.onAc.params.motion['max_shifts'], newoverlaps=None, newstrides=None, upsample_factor_grid=4, upsample_factor_fft=10, show_movie=False, max_deviation_rigid=self.onAc.params. motion['max_deviation_rigid'], add_to_movie=0, shifts_opencv=True, gSig_filt=None, use_cuda=False, border_nan='copy')[:2] else: frame_cor, shift = motion_correct_iteration_fast( frame, templ, self.max_shifts_online, self.max_shifts_online) self.onAc.estimates.shifts.append(shift) else: frame_cor = frame if self.onAc.params.get('online', 'normalize'): frame_cor = frame_cor / self.onAc.img_norm self.procFrame_time.append([time.time() - t]) return frame_cor
' new components were added. Total number of components is ' + str(cnm2.Ab.shape[-1] - gnb)) old_comps = cnm2.N t1 = time() # count time only for the processing part frame_ = frame.copy().astype(np.float32) # if ds_factor > 1: frame_ = cv2.resize(frame_, img_norm.shape[::-1]) # downsampling frame_ -= img_min # make data non-negative if mot_corr: # motion correct templ = cnm2.Ab.dot(cnm2.C_on[:cnm2.M, t - 1]).reshape( cnm2.dims, order='F') * img_norm frame_cor, shift = motion_correct_iteration_fast( frame_, templ, max_shift, max_shift) shifts.append(shift) else: templ = None frame_cor = frame_ frame_cor = frame_cor / img_norm # normalize data-frame cnm2.fit_next(t, frame_cor.reshape( -1, order='F')) # run OnACID on this frame tottime.append(time() - t1) # store time t += 1 if t % 1000 == 0 and plot_contours_flag: pl.cla() A = cnm2.Ab[:, cnm2.gnb:]
raise Exception('Frame ' + str(count) + ' contains nan') if t % 10 == 0: print(t) count = 0 for (x0, x1, y0, y1, frame) in sliding_window_new(whole_frame, overlaps, strides): if count != 0: continue cnm2 = cnms_2[count] # print(count) img_norm = imgs_norm[count] t1 = time() frame_ = frame.copy().astype(np.float32) frame_ -= img_min templ = cnm2.Ab.dot( cnm2.C_on[:, t - 1]).reshape(cnm2.dims, order='F') * img_norm frame_cor, shift = motion_correct_iteration_fast( frame_, templ, max_shifts[0], max_shifts[1]) shifts.append(shift) frame_cor = frame_cor / img_norm cnm2.fit_next(t, frame_cor.reshape(-1, order='F')) tottime.append(time() - t1) cnm_res[count] = cnm2 count += 1 # break t += 1 # A,b = cnm2.Ab[:,:-1],cnm2.Ab[:,-1].toarray() # C, f = cnm2.C_on[:-1,:], cnm2.C_on[-1:,:] # noisyC=cnm2.noisyC # pl.cla() # crd = cm.utils.visualization.plot_contours(A, Cn, thr=0.9)
def main(): pass # For compatibility between running under Spyder and the CLI #%% download and list all files to be processed # folder inside ./example_movies where files will be saved fld_name = 'Mesoscope' download_demo('Tolias_mesoscope_1.hdf5', fld_name) download_demo('Tolias_mesoscope_2.hdf5', fld_name) download_demo('Tolias_mesoscope_3.hdf5', fld_name) # folder where files are located folder_name = os.path.join(caiman_datadir(), 'example_movies', fld_name) extension = 'hdf5' # extension of files # read all files to be processed fls = glob.glob(folder_name + '/*' + extension) # your list of files should look something like this print(fls) #%% Set up some parameters # frame rate (Hz) fr = 15 # approximate length of transient event in seconds decay_time = 0.5 # expected half size of neurons gSig = (3, 3) # order of AR indicator dynamics p = 1 # minimum SNR for accepting new components min_SNR = 2.5 # correlation threshold for new component inclusion rval_thr = 0.85 # spatial downsampling factor (increases speed but may lose some fine structure) ds_factor = 1 # number of background components gnb = 2 # recompute gSig if downsampling is involved gSig = tuple(np.ceil(np.array(gSig) / ds_factor).astype('int')) # flag for online motion correction mot_corr = True # maximum allowed shift during motion correction max_shift = np.ceil(10. / ds_factor).astype('int') # set up some additional supporting parameters needed for the algorithm (these are default values but change according to dataset characteristics) # number of shapes to be updated each time (put this to a finite small value to increase speed) max_comp_update_shape = np.inf # number of files used for initialization init_files = 1 # number of files used for online online_files = len(fls) - 1 # number of frames for initialization (presumably from the first file) initbatch = 200 # maximum number of expected components used for memory pre-allocation (exaggerate here) expected_comps = 300 # initial number of components K = 2 # number of timesteps to consider when testing new neuron candidates N_samples = np.ceil(fr * decay_time) # exceptionality threshold thresh_fitness_raw = scipy.special.log_ndtr(-min_SNR) * N_samples # number of passes over the data epochs = 2 # upper bound for number of frames in each file (used right below) len_file = 1000 # total length of all files (if not known use a large number, then truncate at the end) T1 = len(fls) * len_file * epochs #%% Initialize movie # load only the first initbatch frames and possibly downsample them if ds_factor > 1: Y = cm.load(fls[0], subindices=slice(0, initbatch, None)).astype( np.float32).resize(1. / ds_factor, 1. / ds_factor) else: Y = cm.load(fls[0], subindices=slice(0, initbatch, None)).astype(np.float32) if mot_corr: # perform motion correction on the first initbatch frames mc = Y.motion_correct(max_shift, max_shift) Y = mc[0].astype(np.float32) borders = np.max(mc[1]) else: Y = Y.astype(np.float32) # minimum value of movie. Subtract it to make the data non-negative img_min = Y.min() Y -= img_min img_norm = np.std(Y, axis=0) # normalizing factor to equalize the FOV img_norm += np.median(img_norm) Y = Y / img_norm[None, :, :] # normalize data _, d1, d2 = Y.shape dims = (d1, d2) # dimensions of FOV Yr = Y.to_2D().T # convert data into 2D array Cn_init = Y.local_correlations(swap_dim=False) # compute correlation image #pl.imshow(Cn_init) #pl.title('Correlation Image on initial batch') #pl.colorbar() bnd_Y = np.percentile(Y, (0.001, 100 - 0.001)) # plotting boundaries for Y #%% initialize OnACID with bare initialization cnm_init = bare_initialization(Y[:initbatch].transpose(1, 2, 0), init_batch=initbatch, k=K, gnb=gnb, gSig=gSig, p=p, minibatch_shape=100, minibatch_suff_stat=5, update_num_comps=True, rval_thr=rval_thr, thresh_fitness_raw=thresh_fitness_raw, batch_update_suff_stat=True, max_comp_update_shape=max_comp_update_shape, deconv_flag=False, use_dense=False, simultaneously=False, n_refit=0) #%% Plot initialization results crd = plot_contours(cnm_init.A.tocsc(), Cn_init, thr=0.9) A, C, b, f, YrA, sn = cnm_init.A, cnm_init.C, cnm_init.b, cnm_init.f, cnm_init.YrA, cnm_init.sn view_patches_bar(Yr, scipy.sparse.coo_matrix(A.tocsc()[:, :]), C[:, :], b, f, dims[0], dims[1], YrA=YrA[:, :], img=Cn_init) bnd_AC = np.percentile(A.dot(C), (0.001, 100 - 0.005)) bnd_BG = np.percentile(b.dot(f), (0.001, 100 - 0.001)) #%% create a function for plotting results in real time if needed def create_frame(cnm2, img_norm, captions): A, b = cnm2.Ab[:, cnm2.gnb:], cnm2.Ab[:, :cnm2.gnb].toarray() C, f = cnm2.C_on[cnm2.gnb:cnm2.M, :], cnm2.C_on[:cnm2.gnb, :] # inferred activity due to components (no background) frame_plot = (frame_cor.copy() - bnd_Y[0]) / np.diff(bnd_Y) comps_frame = A.dot(C[:, t - 1]).reshape(cnm2.dims, order='F') bgkrnd_frame = b.dot(f[:, t - 1]).reshape( cnm2.dims, order='F') # denoised frame (components + background) denoised_frame = comps_frame + bgkrnd_frame denoised_frame = (denoised_frame.copy() - bnd_Y[0]) / np.diff(bnd_Y) comps_frame = (comps_frame.copy() - bnd_AC[0]) / np.diff(bnd_AC) if show_residuals: #all_comps = np.reshape(cnm2.Yres_buf.mean(0), cnm2.dims, order='F') all_comps = np.reshape(cnm2.mean_buff, cnm2.dims, order='F') all_comps = np.minimum(np.maximum(all_comps, 0) * 2 + 0.25, 255) else: all_comps = np.array(A.sum(-1)).reshape(cnm2.dims, order='F') # spatial shapes frame_comp_1 = cv2.resize( np.concatenate([frame_plot, all_comps * 1.], axis=-1), (2 * np.int(cnm2.dims[1] * resize_fact), np.int(cnm2.dims[0] * resize_fact))) frame_comp_2 = cv2.resize( np.concatenate([comps_frame, denoised_frame], axis=-1), (2 * np.int(cnm2.dims[1] * resize_fact), np.int(cnm2.dims[0] * resize_fact))) frame_pn = np.concatenate([frame_comp_1, frame_comp_2], axis=0).T vid_frame = np.repeat(frame_pn[:, :, None], 3, axis=-1) vid_frame = np.minimum((vid_frame * 255.), 255).astype('u1') if show_residuals and cnm2.ind_new: add_v = np.int(cnm2.dims[1] * resize_fact) for ind_new in cnm2.ind_new: cv2.rectangle(vid_frame, (int(ind_new[0][1] * resize_fact), int(ind_new[1][1] * resize_fact) + add_v), (int(ind_new[0][0] * resize_fact), int(ind_new[1][0] * resize_fact) + add_v), (255, 0, 255), 2) cv2.putText(vid_frame, captions[0], (5, 20), fontFace=5, fontScale=0.8, color=(0, 255, 0), thickness=1) cv2.putText(vid_frame, captions[1], (np.int(cnm2.dims[0] * resize_fact) + 5, 20), fontFace=5, fontScale=0.8, color=(0, 255, 0), thickness=1) cv2.putText(vid_frame, captions[2], (5, np.int(cnm2.dims[1] * resize_fact) + 20), fontFace=5, fontScale=0.8, color=(0, 255, 0), thickness=1) cv2.putText(vid_frame, captions[3], (np.int(cnm2.dims[0] * resize_fact) + 5, np.int(cnm2.dims[1] * resize_fact) + 20), fontFace=5, fontScale=0.8, color=(0, 255, 0), thickness=1) cv2.putText(vid_frame, 'Frame = ' + str(t), (vid_frame.shape[1] // 2 - vid_frame.shape[1] // 10, vid_frame.shape[0] - 20), fontFace=5, fontScale=0.8, color=(0, 255, 255), thickness=1) return vid_frame #%% Prepare object for OnACID cnm2 = deepcopy(cnm_init) save_init = False # flag for saving initialization object. Useful if you want to check OnACID with different parameters but same initialization if save_init: cnm_init.dview = None save_object(cnm_init, fls[0][:-4] + '_DS_' + str(ds_factor) + '.pkl') cnm_init = load_object(fls[0][:-4] + '_DS_' + str(ds_factor) + '.pkl') path_to_cnn_residual = os.path.join(caiman_datadir(), 'model', 'cnn_model_online.h5') cnm2._prepare_object(np.asarray(Yr), T1, expected_comps, idx_components=None, min_num_trial=3, max_num_added=3, path_to_model=path_to_cnn_residual, sniper_mode=False, use_peak_max=False, q=0.5) cnm2.thresh_CNN_noisy = 0.5 #%% Run OnACID and optionally plot results in real time epochs = 1 cnm2.Ab_epoch = [] # save the shapes at the end of each epoch t = cnm2.initbatch # current timestep tottime = [] Cn = Cn_init.copy() # flag for removing components with bad shapes remove_flag = False T_rm = 650 # remove bad components every T_rm frames rm_thr = 0.1 # CNN classifier removal threshold # flag for plotting contours of detected components at the end of each file plot_contours_flag = False # flag for showing results video online (turn off flags for improving speed) play_reconstr = True # flag for saving movie (file could be quite large..) save_movie = False movie_name = os.path.join( folder_name, 'sniper_meso_0.995_new.avi') # name of movie to be saved resize_fact = 1.2 # image resizing factor if online_files == 0: # check whether there are any additional files process_files = fls[:init_files] # end processing at this file init_batc_iter = [initbatch] # place where to start end_batch = T1 else: process_files = fls[:init_files + online_files] # additional files # where to start reading at each file init_batc_iter = [initbatch] + [0] * online_files shifts = [] show_residuals = True if show_residuals: caption = 'Mean Residual Buffer' else: caption = 'Identified Components' captions = ['Raw Data', 'Inferred Activity', caption, 'Denoised Data'] if save_movie and play_reconstr: fourcc = cv2.VideoWriter_fourcc('8', 'B', 'P', 'S') # fourcc = cv2.VideoWriter_fourcc(*'XVID') out = cv2.VideoWriter( movie_name, fourcc, 30.0, tuple([int(2 * x * resize_fact) for x in cnm2.dims])) for iter in range(epochs): if iter > 0: # if not on first epoch process all files from scratch process_files = fls[:init_files + online_files] init_batc_iter = [0] * (online_files + init_files) # np.array(fls)[np.array([1,2,3,4,5,-5,-4,-3,-2,-1])]: for file_count, ffll in enumerate(process_files): print('Now processing file ' + ffll) Y_ = cm.load(ffll, subindices=slice(init_batc_iter[file_count], T1, None)) # update max-correlation (and perform offline motion correction) just for illustration purposes if plot_contours_flag: if ds_factor > 1: Y_1 = Y_.resize(1. / ds_factor, 1. / ds_factor, 1) else: Y_1 = Y_.copy() if mot_corr: templ = (cnm2.Ab.data[:cnm2.Ab.indptr[1]] * cnm2.C_on[0, t - 1]).reshape(cnm2.dims, order='F') * img_norm newcn = (Y_1 - img_min).motion_correct( max_shift, max_shift, template=templ)[0].local_correlations(swap_dim=False) Cn = np.maximum(Cn, newcn) else: Cn = np.maximum(Cn, Y_1.local_correlations(swap_dim=False)) old_comps = cnm2.N # number of existing components for frame_count, frame in enumerate(Y_): # now process each file if np.isnan(np.sum(frame)): raise Exception('Frame ' + str(frame_count) + ' contains nan') if t % 100 == 0: print( 'Epoch: ' + str(iter + 1) + '. ' + str(t) + ' frames have beeen processed in total. ' + str(cnm2.N - old_comps) + ' new components were added. Total number of components is ' + str(cnm2.Ab.shape[-1] - gnb)) old_comps = cnm2.N t1 = time() # count time only for the processing part frame_ = frame.copy().astype(np.float32) # if ds_factor > 1: frame_ = cv2.resize(frame_, img_norm.shape[::-1]) # downsampling frame_ -= img_min # make data non-negative if mot_corr: # motion correct templ = cnm2.Ab.dot(cnm2.C_on[:cnm2.M, t - 1]).reshape( cnm2.dims, order='F') * img_norm frame_cor, shift = motion_correct_iteration_fast( frame_, templ, max_shift, max_shift) shifts.append(shift) else: templ = None frame_cor = frame_ frame_cor = frame_cor / img_norm # normalize data-frame cnm2.fit_next(t, frame_cor.reshape( -1, order='F')) # run OnACID on this frame # store time tottime.append(time() - t1) t += 1 if t % T_rm == 0 and remove_flag: prd, _ = evaluate_components_CNN(cnm2.Ab[:, gnb:], dims, gSig) ind_rem = np.where(prd[:, 1] < rm_thr)[0].tolist() cnm2.remove_components(ind_rem) print('Removing ' + str(len(ind_rem)) + ' components') if t % 1000 == 0 and plot_contours_flag: pl.cla() A = cnm2.Ab[:, cnm2.gnb:] # update the contour plot every 1000 frames crd = cm.utils.visualization.plot_contours(A, Cn, thr=0.9) pl.pause(1) if play_reconstr: # generate movie with the results vid_frame = create_frame(cnm2, img_norm, captions) if save_movie: out.write(vid_frame) if t - initbatch < 100: #for rp in np.int32(np.ceil(np.exp(-np.arange(1,100)/30)*20)): for rp in range(len(cnm2.ind_new) * 2): out.write(vid_frame) cv2.imshow('frame', vid_frame) if t - initbatch < 100: for rp in range(len(cnm2.ind_new) * 2): cv2.imshow('frame', vid_frame) if cv2.waitKey(1) & 0xFF == ord('q'): break print('Cumulative processing speed is ' + str((t - initbatch) / np.sum(tottime))[:5] + ' frames per second.') # save the shapes at the end of each epoch cnm2.Ab_epoch.append(cnm2.Ab.copy()) if save_movie: out.release() cv2.destroyAllWindows() #%% save results (optional) save_results = False if save_results: np.savez('results_analysis_online_MOT_CORR.npz', Cn=Cn, Ab=cnm2.Ab, Cf=cnm2.C_on, b=cnm2.b, f=cnm2.f, dims=cnm2.dims, tottime=tottime, noisyC=cnm2.noisyC, shifts=shifts) #%% extract results from the objects and do some plotting A, b = cnm2.Ab[:, cnm2.gnb:], cnm2.Ab[:, :cnm2.gnb].toarray() C, f = cnm2.C_on[cnm2.gnb:cnm2.M, t - t // epochs:t], cnm2.C_on[:cnm2.gnb, t - t // epochs:t] noisyC = cnm2.noisyC[:, t - t // epochs:t] b_trace = [osi.b for osi in cnm2.OASISinstances] if hasattr( cnm2, 'OASISinstances') else [0] * C.shape[0] pl.figure() crd = cm.utils.visualization.plot_contours(A, Cn, thr=0.9) view_patches_bar(Yr, scipy.sparse.coo_matrix(A.tocsc()[:, :]), C[:, :], b, f, dims[0], dims[1], YrA=noisyC[cnm2.gnb:cnm2.M] - C, img=Cn)
def __fit_next_frame(self, frame, t, model_LN=None, out=None): ssub_B = self.params.get('init', 'ssub_B') * self.params.get( 'init', 'ssub') d1, d2 = self.params.get('data', 'dims') max_shifts_online = self.params.get('online', 'max_shifts_online') if model_LN is not None: if self.params.get('ring_CNN', 'remove_activity'): activity = self.estimates.Ab[:, :self.N].dot( self.estimates.C_on[:self.N, t - 1]).reshape(self.params.get( 'data', 'dims'), order='F') if self.params.get('online', 'normalize'): activity *= self.img_norm else: activity = 0. # frame = frame.astype(np.float32) - activity frame = frame - np.squeeze( model_LN.predict( np.expand_dims( np.expand_dims(frame.astype(np.float32) - activity, 0), -1))) frame = np.maximum(frame, 0) t_frame_start = time.time() if np.isnan(np.sum(frame)): raise Exception('Current frame contains NaN') frame_ = frame.copy().astype(np.float32) if self.params.get('online', 'ds_factor') > 1: frame_ = cv2.resize(frame_, self.img_norm.shape[::-1]) if self.params.get('online', 'normalize'): frame_ -= self.img_min # make data non-negative if self.params.get('online', 'motion_correct'): templ = self.estimates.Ab.dot( np.median(self.estimates.C_on[:self.M, t - 51:t - 1], 1)).reshape(self.params.get('data', 'dims'), order='F') #*self.img_norm if self.is1p and self.estimates.W is not None: if ssub_B == 1: B = self.estimates.W.dot((frame_ - templ).flatten( order='F') - self.estimates.b0) + self.estimates.b0 B = B.reshape(self.params.get('data', 'dims'), order='F') else: b0 = self.estimates.b0.reshape((d1, d2), order='F') #*self.img_norm bc2 = initialization.downscale( frame_ - templ - b0, (ssub_B, ssub_B)).flatten(order='F') Wb = self.estimates.W.dot(bc2).reshape( ((d1 - 1) // ssub_B + 1, (d2 - 1) // ssub_B + 1), order='F') B = b0 + np.repeat(np.repeat(Wb, ssub_B, 0), ssub_B, 1)[:d1, :d2] templ += B if self.params.get('online', 'normalize'): templ *= self.img_norm if self.is1p: templ = high_pass_filter_space(templ, self.params.motion['gSig_filt']) if self.params.get('motion', 'pw_rigid'): frame_cor, shift, _, xy_grid = tile_and_correct( frame_, templ, self.params.motion['strides'], self.params.motion['overlaps'], self.params.motion['max_shifts'], newoverlaps=None, newstrides=None, upsample_factor_grid=4, upsample_factor_fft=10, show_movie=False, max_deviation_rigid=self.params. motion['max_deviation_rigid'], add_to_movie=0, shifts_opencv=True, gSig_filt=None, use_cuda=False, border_nan='copy') else: if self.is1p: frame_orig = frame_.copy() frame_ = high_pass_filter_space( frame_, self.params.motion['gSig_filt']) frame_cor, shift = motion_correct_iteration_fast( frame_, templ, max_shifts_online, max_shifts_online) if self.is1p: M = np.float32([[1, 0, shift[1]], [0, 1, shift[0]]]) frame_cor = cv2.warpAffine(frame_orig, M, frame_.shape[::-1], flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REFLECT) self.estimates.shifts.append(shift) else: templ = None frame_cor = frame_ self.current_frame_cor = frame_cor if self.params.get('online', 'normalize'): frame_cor = frame_cor / self.img_norm self.fit_next(t, frame_cor.reshape(-1, order='F')) if self.fp_detector.method is not None: self.__reject_fp_comps((d1, d2), max_bright=frame_.max())