def plot_contours_evaluated(row = None, session_wise = False): ''' Plot contours for all cells, selected cells and non selected cells, and saves it in figure_path = '/data/interim/component_evaluation/trial_wise/meta/figures/contours/' :param row: one analysis state row ''' index = row.name corr_min = round(eval(row['source_extraction_parameters'])['min_corr'],1) pnr_min = round(eval(row['source_extraction_parameters'])['min_pnr'],1) r_min = eval(row['component_evaluation_parameters'])['rval_thr'] snf_min = eval(row['component_evaluation_parameters'])['min_SNR'] output_source_extraction = eval(row.loc['source_extraction_output']) corr_path = output_source_extraction['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) output_component_evaluation = eval(row.loc['component_evaluation_output']) cnm_file_path = output_component_evaluation['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) figure, axes = plt.subplots(1, 3) axes[0].imshow(cn_filter) axes[1].imshow(cn_filter) axes[2].imshow(cn_filter) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[0].plot(*v.T, c='w') axes[0].set_title('All components') axes[0].set_ylabel('Corr=' + f'{corr_min}' + ', PNR = ' + f'{pnr_min}' + ', PCC = ' + f'{r_min}' + ', SNR =' + f'{snf_min}') idx = cnm.estimates.idx_components coordinates = cm.utils.visualization.get_contours(cnm.estimates.A[:,idx], np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[1].plot(*v.T, c='b') axes[1].set_title('Accepted components') idx_b = cnm.estimates.idx_components_bad coordinates_b = cm.utils.visualization.get_contours(cnm.estimates.A[:,idx_b], np.shape(cn_filter), 0.2, 'max') for c in coordinates_b: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[2].plot(*v.T, c='r') axes[2].set_title('Rejected components') figure_path = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/interim/component_evaluation/trial_wise/meta/figures/contours/' if session_wise: figure_path = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/interim/component_evaluation/session_wise/meta/figures/contours/' figure_name = figure_path + db.create_file_name(5,index) + '.png' figure.savefig(figure_name) return figure
def plot_crispness_for_parameters(selected_rows = None): ''' This function plots crispness for all the selected rows motion correction states. The idea is to compare crispness results :param selected_rows: analysis states for which crispness is required to be ploted :return: figure that is also saved ''' crispness_mean_original,crispness_corr_original, crispness_mean, crispness_corr = metrics.compare_crispness(selected_rows) total_states_number = len(selected_rows) fig, axes = plt.subplots(1,2) axes[0].set_title('Summary image = Mean') axes[0].plot(np.arange(1,total_states_number,1),crispness_mean_original) axes[0].plot(np.arange(1,total_states_number,1),crispness_mean) axes[0].legend(('Original', 'Motion_corrected')) axes[0].set_ylabel('Crispness') #axes[0].set_xlabel('#') axes[1].set_title('Summary image = Corr') axes[1].plot(np.arange(1,total_states_number,1),crispness_corr_original) axes[1].plot(np.arange(1,total_states_number,1),crispness_corr) axes[1].legend(('Original', 'Motion_corrected')) axes[1].set_ylabel('Crispness') #axes[0].set_xlabel('#') # Get output file paths index = selected_rows.iloc[0].name data_dir = 'data/interim/motion_correction/' step_index = db.get_step_index('motion_correction') file_name = db.create_file_name(step_index, index) output_meta_crispness = data_dir + f'meta/figures/crispness/{file_name}.png' fig.savefig(output_meta_crispness) return fig
def plot_source_extraction_result(mouse_row_new): ''' Generates and saves a contour plot and a trace plot for the specific mouse_row ''' corr_min = round(eval(mouse_row_new['source_extraction_parameters'])['min_corr'], 1) pnr_min = round(eval(mouse_row_new['source_extraction_parameters'])['min_pnr'], 1) output_source_extraction = eval(mouse_row_new.loc['source_extraction_output']) corr_path = output_source_extraction['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) cnm_file_path = output_source_extraction['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) figure, axes = plt.subplots(1) axes.imshow(cn_filter) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes.plot(*v.T, c='w') axes.set_title('min_corr = ' + f'{corr_min}') axes.set_ylabel('min_pnr = ' + f'{pnr_min}') fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/contours/' file_name = db.create_file_name(3, mouse_row_new.name) figure.savefig(fig_dir + file_name + '.png') ## up to here fig, ax = plt.subplots(1) C = cnm.estimates.C C[0] += C[0].min() for i in range(1, len(C)): C[i] += C[i].min() + C[:i].max() ax.plot(C[i]) ax.set_xlabel('t [frames]') ax.set_yticks([]) ax.set_ylabel('activity') fig.set_size_inches([10., .3 * len(C)]) fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/traces/' fig_name = fig_dir + db.create_file_name(3, mouse_row_new.name) + '.png' fig.savefig(fig_name) return
def plot_traces_multiple_evaluated(row = None, session_wise = False): ''' Plots different versions of contour images that change the inicialization parameters for source extraccion. The idea is to see the impact of different seed selection in the final source extraction result. :param row: one analysis state row :return: figure ''' corr_min = round(eval(row['source_extraction_parameters'])['min_corr'],1) pnr_min = round(eval(row['source_extraction_parameters'])['min_pnr'],1) r_min = eval(row['component_evaluation_parameters'])['rval_thr'] snf_min = eval(row['component_evaluation_parameters'])['min_SNR'] output_source_extraction = eval(row.loc['source_extraction_output']) corr_path = output_source_extraction['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) cnm_file_path = output_source_extraction['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) C = cnm.estimates.C output_component_evaluation = eval(row.loc['component_evaluation_output']) cnm_file_path = output_component_evaluation['main'] cnm_eval = load_CNMF(db.get_file(cnm_file_path)) idx = cnm_eval.estimates.idx_components idx_b = cnm_eval.estimates.idx_components_bad fig, ax = plt.subplots(1) C[0] += C[0].min() for i in range(1, len(C)): C[i] += C[i].min() + C[:i].max() if i in idx_b: color = 'red' else: color = 'blue' ax.plot(C[i],color = color) ax.set_xlabel('t [frames]') ax.set_yticks([]) ax.set_ylabel('activity') ax.set_title('Corr=' + f'{corr_min}' + ', PNR = ' + f'{pnr_min}' + ', PCC = ' + f'{r_min}' + ', SNR =' + f'{snf_min}') fig.set_size_inches([10., .3 * len(C)]) fig_dir = 'data/interim/component_evaluation/trial_wise/meta/figures/traces/' if session_wise: fig_dir = 'data/interim/component_evaluation/session_wise/meta/figures/traces/' fig_name = fig_dir + db.create_file_name(5,row.name) + '.png' fig.savefig(fig_name) return
def plot_source_extraction_result_specific_cell(mouse_row_new, cell_number): ''' (Still need to be finished) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! THERE IS AN ERROR IN THE In the first plot shows correlation image and contour of the selected neurons. In the second plot shows the traces for the selected neurons. :param mouse_row_new: data base row :param cell_number: array with the cells that are selected to be ploted :return: None ''' corr_min = round(eval(mouse_row_new['source_extraction_parameters'])['min_corr'], 1) pnr_min = round(eval(mouse_row_new['source_extraction_parameters'])['min_pnr'], 1) output_source_extraction = eval(mouse_row_new.loc['source_extraction_output']) corr_path = output_source_extraction['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) cnm_file_path = output_source_extraction['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) f, (a0, a1) = plt.subplots(2, 1, gridspec_kw={'height_ratios': [3, 1]}) a0.imshow(cn_filter) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for i in cell_number: v = coordinates[i]['coordinates'] coordinates[i]['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] a0.plot(*v.T, c='w') a0.set_title('Contour Plot') fig, ax = plt.subplots(1) C = cnm.estimates.C C[0] += C[0].min() for i in range(cell_number): C[i] += C[i].min() + C[:i].max() a1.plot(C[i]) a1.set_xlabel('t [frames]') a1.set_yticks([]) a1.set_title('Calcium Traces') fig.set_size_inches([10., .3 * len(C)]) fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/' fig_name = fig_dir + db.create_file_name(3, mouse_row_new.name) + '_example.png' f.savefig(fig_name) return
def plot_multiple_contours_session_wise(selected_rows, version = None , corr_array = None, pnr_array = None): ''' Plots different versions of contour images that change the initialization parameters for source extraction. The idea is to see the impact of different seed selection in the final source extraction result. :param selected_rows: all analysis state selected :param version: array containing the version numbers of source extraction that will be plotted :param corr_array: array of the same length of version and pnr_array containing the min_corr values for those versions :param pnr_array: array of the same length of version and corr_array containing the min_pnr values for those versions :return: figure ''' states_df = db.open_analysis_states_database() figure, axes = plt.subplots(len(corr_array), len(pnr_array), figsize=(15, 15)) color = ['w','b','r','m','c'] for row in range(len(selected_rows)): mouse_row = selected_rows.iloc[row] index = mouse_row.name output = eval(mouse_row.loc['source_extraction_output']) corr_path = output['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) for ii in range(corr_array.shape[0]): for jj in range(pnr_array.shape[0]): axes[ii, jj].imshow(cn_filter) new_row = db.select(states_df, 'component_evaluation', mouse=index[0], session=index[1], trial=index[2], is_rest=index[3], cropping_v=index[5], motion_correction_v=index[6], source_extraction_v=version[ii * len(pnr_array) + jj]) new_row = new_row.iloc[0] output = eval(new_row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[ii, jj].plot(*v.T, c = color[row]) axes[ii, jj].set_title('min_corr = ' + f'{round(corr_array[ii],2)}') axes[ii, jj].set_ylabel('min_pnr = ' + f'{round(pnr_array[jj],2)}') fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/contours/' fig_name = fig_dir + db.create_file_name(3, new_row.name)+'_corr_min' + f'{round(corr_array[0],1)}'+ '_pnr_min'+f'{round(pnr_array[0],1)}' + '_all.png' figure.savefig(fig_name) return figure
def plot_multiple_contours(rows, version = None , corr_array = None, pnr_array = None,session_wise = False): ''' Plots different versions of contour images that change the initialization parameters for source extraction. The idea is to see the impact of different seed selection in the final source extraction result. :param row: one analysis state row :param version: array containing the version numbers of source extraction that will be plotted :param corr_array: array of the same length of version and pnr_array containing the min_corr values for those versions :param pnr_array: array of the same length of version and corr_array containing the min_pnr values for those versions :return: figure ''' figure, axes = plt.subplots(len(corr_array), len(pnr_array), figsize=(15, 15)) for ii in range(corr_array.shape[0]): for jj in range(pnr_array.shape[0]): version_number = ii *corr_array.shape[0] + jj + 1 if version_number in version: new_row = rows.query('(source_extraction_v == ' + f'{version_number}' + ')') new_row = new_row.iloc[0] output = eval(new_row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) corr_path = output['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) axes[ii, jj].imshow(cn_filter) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[ii, jj].plot(*v.T, c='w') axes[ii, jj].set_title('min_corr = ' + f'{round(corr_array[ii],2)}') axes[ii, jj].set_ylabel('min_pnr = ' + f'{round(pnr_array[jj],2)}') fig_dir = 'data/interim/source_extraction/trial_wise/meta/figures/contours/' if session_wise: fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/contours/' fig_name = fig_dir + db.create_file_name(3, new_row.name)+'_corr_min' + f'{round(corr_array[0],1)}'+ '_pnr_min'+f'{round(pnr_array[0],1)}' + '_.png' figure.savefig(fig_name) return figure
def plot_session_contours(selected_rows, version = None , corr_array = None, pnr_array = None): ''' Plots different versions of contour images that change the initialization parameters for source extraction. The idea is to see the impact of different seed selection in the final source extraction result. :param selected_rows: rows corresponding to different trials :param version: array containing the version numbers of source extraction that will be plotted :param corr_array: array of the same length of version and pnr_array containing the min_corr values for those versions :param pnr_array: array of the same length of version and corr_array containing the min_pnr values for those versions :return: (saves multiple figures) ''' for ii in range(corr_array.shape[0]): for jj in range(pnr_array.shape[0]): figure, axes = plt.subplots(len(selected_rows) / 5, 5, figsize=(50, 10*len(selected_rows) / 5)) version_rows = selected_rows.query('(source_extraction_v == ' + f'{ii * len(corr_array.shape[0] + jj)}' + ')') for day in range(len(selected_rows)/5): for trial in range(5): new_row = version_rows.iloc[day*5+trial] output = eval(new_row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) corr_path = output['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) #axes[i].imshow(np.clip(cn_filter, min_corr, max_corr), cmap='viridis') axes[day,trial].imshow(cn_filter) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[day,trial].plot(*v.T, c='w') axes[day,trial].set_title('Trial = ' + f'{i+1}',fontsize=30) axes[day,trial].set_xlabel('#cells = ' + f'{cnm.estimates.A.shape[1]}',fontsize=30) figure.suptitle('min_corr = ' + f'{round(corr_array[ii],2)}' + 'min_pnr = ' + f'{round(pnr_array[jj],2)}', fontsize=50) fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/contours/' fig_name = fig_dir + db.create_file_name(3, new_row.name)+'_version_' + f'{version[ii*len(pnr_array)+jj]}'+'.png' figure.savefig(fig_name) return
def plot_traces_multiple(rows, version = None , corr_array = None, pnr_array = None, session_wise = False): ''' Plots different versions of contour images that change the inicialization parameters for source extraccion. The idea is to see the impact of different seed selection in the final source extraction result. :param row: one analysis state row :param version: array containing the version numbers of source extraction that will be ploted :param corr_array: array of the same length of version and pnr_array containing the min_corr values for those versions :param pnr_array: array of the same length of version and corr_array containing the min_pnr values for those versions :param: session_wise bool that indicates where the figure is save :return: None ''' for ii in range(corr_array.shape[0]): for jj in range(pnr_array.shape[0]): fig, ax = plt.subplots(1) new_row = rows.query('(source_extraction_v == ' + f'{ii *corr_array.shape[0] + jj + 1}' + ')') new_row = new_row.iloc[0] output = eval(new_row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) C = cnm.estimates.C idx_components = cnm.estimates.idx_components C[0] += C[0].min() for i in range(1, len(C)): C[i] += C[i].min() + C[:i].max() ax.plot(C[i]) ax.set_xlabel('t [frames]') ax.set_yticks([]) ax.set_ylabel('activity') fig.set_size_inches([10., .3 * len(C)]) fig_dir = 'data/interim/source_extraction/trial_wise/meta/figures/traces/' if session_wise: fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/traces/' fig_name = fig_dir + db.create_file_name(3,new_row.name) + '_corr_min' + f'{round(corr_array[ii], 1)}' + '_pnr_min' + f'{round(pnr_array[jj], 1)}' + '_.png' fig.savefig(fig_name) return
def get_corr_pnr_path(index, gSig_abs=None): fname = db.create_file_name(2, index) os.chdir(os.environ['PROJECT_DIR']) corr_dir = 'data/interim/source_extraction/trial_wise/meta/corr' corr_path = None for path in os.listdir(corr_dir): if fname in path: if gSig_abs == None: corr_path = os.path.join(corr_dir, path) else: if path[-5] == str(gSig_abs): corr_path = os.path.join(corr_dir, path) pnr_dir = 'data/interim/source_extraction/trial_wise/meta/pnr' pnr_path = None for path in os.listdir(pnr_dir): if fname in path: if gSig_abs == None: pnr_path = os.path.join(pnr_dir, path) else: if path[-5] == str(gSig_abs): pnr_path = os.path.join(pnr_dir, path) return corr_path, pnr_path
def get_fig_gSig_filt_vals(row, gSig_filt_vals): ''' Plot original cropped frame and several versions of spatial filtering for comparison :param row: analisis state row for which the filtering is computed :param gSig_filt_vals: array containing size of spatial filters that will be applyed :return: figure ''' output = row['cropping_output'] cropped_file = eval(output)['main'] m = cm.load(cropped_file) temp = cm.motion_correction.bin_median(m) N = len(gSig_filt_vals) fig, axes = plt.subplots(int(math.ceil((N + 1) / 2)), 2) axes[0, 0].imshow(temp, cmap='gray') axes[0, 0].set_title('unfiltered') axes[0, 0].axis('off') for i in range(0, N): gSig_filt = gSig_filt_vals[i] m_filt = [high_pass_filter_space(m_, (gSig_filt, gSig_filt)) for m_ in m] temp_filt = cm.motion_correction.bin_median(m_filt) axes.flatten()[i + 1].imshow(temp_filt, cmap='gray') axes.flatten()[i + 1].set_title(f'gSig_filt = {gSig_filt}') axes.flatten()[i + 1].axis('off') if N + 1 != axes.size: for i in range(N + 1, axes.size): axes.flatten()[i].axis('off') # Get output file paths index = row.name data_dir = 'data/interim/motion_correction/' step_index = db.get_step_index('motion_correction') file_name = db.create_file_name(step_index, index) output_meta_gSig_filt = data_dir + f'meta/figures/frame_gSig_filt/{file_name}.png' fig.savefig(output_meta_gSig_filt) return fig
def plot_temporal_evolution(row,session_wise = False): ''' After decoding this plots the time evolution of some pixel values in the ROI, the histogram if pixel values and the ROI with the mark of the position for the randomly selected pixels If non specified it uses the trial video, if not it uses the aligned version of the videos. If alignement version == 2, then it uses the equalized version ''' if session_wise: output = row['alignment_output'] else: output = row['decoding_output'] decoded_file = eval(output)['main'] if row.name[7] == 2: decoded_file = eval(output)['equalization']['main'] movie_original = cm.load(decoded_file) figure = plt.figure(constrained_layout=True) gs = figure.add_gridspec(5, 6) figure_ax1 = figure.add_subplot(gs[0:2, 0:3]) figure_ax1.set_title('ROI: ' + f"mouse_{row.name[0]}", fontsize = 15) figure_ax1.set_yticks([]) figure_ax1.set_xticks([]) figure_ax2 = figure.add_subplot(gs[2:5, 0:3]) figure_ax2.set_xlabel('Time [s]', fontsize = 15) figure_ax2.set_ylabel('Pixel value', fontsize = 15) figure_ax2.set_title('Temporal Evolution', fontsize = 15) figure_ax2.set_ylim((400,2000)) figure_ax1.imshow(movie_original[0,:,:], cmap = 'gray') color = ['b', 'r' , 'g', 'c', 'm'] for i in range(5): x = randrange(movie_original.shape[1]-5)+5 y = randrange(movie_original.shape[2]-5)+5 [x_, _x, y_, _y] = [x-5,x+5,y-5,y+5] rect = Rectangle((y_, x_), _y - y_, _x - x_, fill=False, color=color[i], linestyle='-', linewidth=2) figure_ax1.add_patch(rect) figure_ax2.plot(np.arange(0,movie_original.shape[0],)/10, movie_original[:,x,y], color = color[i]) figure_ax_i = figure.add_subplot(gs[i, 4:]) figure_ax_i.hist(movie_original[:,x,y],50, color = color[i]) figure_ax_i.set_xlim((400,1100)) figure_ax_i.set_ylabel('#') figure_ax_i.set_xlabel('Pixel value') path = os.environ['DATA_DIR']+ '/data/interim/decoding/meta/' name = db.create_file_name(1, row.name) if session_wise: path = os.environ['DATA_DIR'] + '/data/interim/alignment/meta/' name = db.create_file_name(4,row.name) if row.name[7] == 2: name = db.create_file_name(4, row.name) figure.savefig(path + name + '.png') figure2, axes = plt.subplots(1) mean_movie = np.mean(movie_original,axis = (1,2) ) axes.plot(np.arange(0, movie_original.shape[0], ) / 10, mean_movie, color='b') axes.set_xlabel('Time [s]', fontsize = 15) axes.set_ylabel('Mean Pixel Value', fontsize = 15) axes.set_title('Temporal Evolution', fontsize = 15) axes.set_ylim((400,2000)) if row.name[7] == 2: name = db.create_file_name(4, row.name) figure2.savefig(path + name + '_mean.png') return
distance_list = [] for i in range(len(A_center_of_mass_list_x)): for j in range(i + 1, len(A_center_of_mass_list_x)): x1 = A_center_of_mass_list_x[i] x2 = A_center_of_mass_list_x[j] y1 = A_center_of_mass_list_y[i] y2 = A_center_of_mass_list_y[j] distance = math.sqrt((x1 - x2)**2 + (y1 - y2)**2) distance_list.append(distance) figure, axes = plt.subplots(1) axes.hist(distance_list, bins=50) axes.set_xlabel('Distance [pixels]', fontsize=12) axes.set_ylabel('Number of pairs', fontsize=12) figure.suptitle('Distance between center of mass', fontsize=20) figure_name = db.create_file_name(5, row.name) figure.savefig(figure_path + figure_name + '.png') figure1, axes = plt.subplots(1) for i in range(1, 126): new_vector_x = [] new_vector_y = [] for j in range(len(A_center_of_mass_list_y)): if trial_belonging[j] == i: new_vector_x.append(A_center_of_mass_list_x[j]) new_vector_y.append(A_center_of_mass_list_y[j]) axes.scatter(new_vector_y, new_vector_x) ## this part computes the pcc between contours templates for the entire session pcc_list = []
def run_alignmnet(selected_rows, parameters, dview): ''' This is the main function for the alignment step. It applies methods from the CaImAn package used originally in motion correction to do alignment. Args: df: pd.DataFrame A dataframe containing the analysis states you want to have aligned. parameters: dict The alignment parameters. dview: object The dview object Returns: df: pd.DataFrame A dataframe containing the aligned analysis states. ''' # Sort the dataframe correctly df = selected_rows.copy() df = df.sort_values(by=paths.multi_index_structure) # Determine the mouse and session of the dataset index = df.iloc[0].name mouse, session, *r = index # alignment_v = index[len(paths.data_structure) + step_index] alignment_v = len(df) alignment_index = (mouse, session, alignment_v) # Determine the output .mmap file name file_name = f'mouse_{mouse}_session_{session}_v{alignment_v}' output_mmap_file_path = os.environ['DATA_DIR'] + f'data/interim/alignment/main/{file_name}.mmap' try: df.reset_index()[['session','trial', 'is_rest']].set_index(['session','trial', 'is_rest'], verify_integrity=True) except ValueError: logging.error('You passed multiple of the same trial in the dataframe df') return df output = { 'meta': { 'analysis': { 'analyst': os.environ['ANALYST'], 'date': datetime.datetime.today().strftime("%m-%d-%Y"), 'time': datetime.datetime.today().strftime("%H:%M:%S") }, 'duration': {} } } # Get necessary parameters motion_correction_parameters_list = [] motion_correction_output_list = [] input_mmap_file_list = [] trial_index_list = [] x_ = [] _x = [] y_ = [] _y = [] for idx, row in df.iterrows(): motion_correction_parameters_list.append(eval(row.loc['motion_correction_parameters'])) motion_correction_output = eval(row.loc['motion_correction_output']) motion_correction_output_list.append(motion_correction_output) input_mmap_file_list.append(motion_correction_output['main']) trial_index_list.append(db.get_trial_name(idx[2], idx[3])) [x1,x2,y1,y2] = motion_correction_output['meta']['cropping_points'] x_.append(x1) _x.append(x2) y_.append(y1) _y.append(y2) new_x1 = max(x_) new_x2 = max(_x) new_y1 = max(y_) new_y2 = max(_y) m_list = [] for i in range(len(input_mmap_file_list)): m = cm.load(input_mmap_file_list[i]) motion_correction_output = eval(df.iloc[i].loc['motion_correction_output']) [x1,x2,y1,y2] = motion_correction_output['meta']['cropping_points'] m = m.crop(new_x1 - x1, new_x2 - x2, new_y1 - y1, new_y2 - y2, 0, 0) m_list.append(m) # Concatenate them using the concat function m_concat = cm.concatenate(m_list, axis=0) data_dir = os.environ['DATA_DIR'] + 'data/interim/alignment/main/' file_name = db.create_file_name(step_index, index) fname= m_concat.save(data_dir + file_name + '.mmap', order='C') #meta_pkl_dict['pw_rigid']['cropping_points'] = [x_, _x, y_, _y] #output['meta']['cropping_points'] = [x_, _x, y_, _y] # Save the movie #fname_tot_els = m_els.save(data_dir + 'main/' + file_name + '_els' + '.mmap', order='C') #logging.info(f'{index} Cropped and saved rigid movie as {fname_tot_els}') # MOTION CORRECTING EACH INDIVIDUAL MOVIE WITH RESPECT TO A TEMPLATE MADE OF THE FIRST MOVIE logging.info(f'{alignment_index} Performing motion correction on all movies with respect to a template made of \ the first movie.') t0 = datetime.datetime.today() # Create a template of the first movie template_index = trial_index_list.index(parameters['make_template_from_trial']) m0 = cm.load(input_mmap_file_list[template_index ]) [x1, x2, y1, y2] = motion_correction_output_list[template_index]['meta']['cropping_points'] m0 = m0.crop(new_x1 - x1, new_x2 - x2, new_y1 - y1, new_y2 - y2, 0, 0) m0_filt = cm.movie( np.array([high_pass_filter_space(m_, parameters['gSig_filt']) for m_ in m0])) template0 = cm.motion_correction.bin_median( m0_filt.motion_correct(5, 5, template=None)[0]) # may be improved in the future # Setting the parameters opts = params.CNMFParams(params_dict=parameters) # Create a motion correction object mc = MotionCorrect(fname, dview=dview, **opts.get_group('motion')) # Perform non-rigid motion correction mc.motion_correct(template=template0, save_movie=True) # Cropping borders x_ = math.ceil(abs(np.array(mc.shifts_rig)[:, 1].max()) if np.array(mc.shifts_rig)[:, 1].max() > 0 else 0) _x = math.ceil(abs(np.array(mc.shifts_rig)[:, 1].min()) if np.array(mc.shifts_rig)[:, 1].min() < 0 else 0) y_ = math.ceil(abs(np.array(mc.shifts_rig)[:, 0].max()) if np.array(mc.shifts_rig)[:, 0].max() > 0 else 0) _y = math.ceil(abs(np.array(mc.shifts_rig)[:, 0].min()) if np.array(mc.shifts_rig)[:, 0].min() < 0 else 0) # Load the motion corrected movie into memory movie= cm.load(mc.fname_tot_rig[0]) # Crop all movies to those border pixels movie.crop(x_, _x, y_, _y, 0, 0) output['meta']['cropping_points'] = [x_, _x, y_, _y] #save motion corrected and cropped movie output_mmap_file_path_tot = movie.save(data_dir + file_name + '.mmap', order='C') logging.info(f'{index} Cropped and saved rigid movie as {output_mmap_file_path_tot}') # Save the path in teh output dictionary output['main'] = output_mmap_file_path_tot # Remove the remaining non-cropped movie os.remove(mc.fname_tot_rig[0]) # Create a timeline and store it timeline = [[trial_index_list[0], 0]] timepoints = [0] for i in range(1, len(m_list)): m = m_list[i] timeline.append([trial_index_list[i], timeline[i - 1][1] + m.shape[0]]) timepoints.append(timepoints[i-1]+ m.shape[0]) timeline_pkl_file_path = os.environ['DATA_DIR'] + f'data/interim/alignment/meta/timeline/{file_name}.pkl' with open(timeline_pkl_file_path,'wb') as f: pickle.dump(timeline,f) output['meta']['timeline'] = timeline_pkl_file_path timepoints.append(movie.shape[0]) dt = int((datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes output['meta']['duration']['concatenation'] = dt logging.info(f'{alignment_index} Performed concatenation. dt = {dt} min.') for idx, row in df.iterrows(): df.loc[idx, 'alignment_output'] = str(output) df.loc[idx, 'alignment_parameters'] = str(parameters) ## modify all motion correction file to the aligned version data_dir = os.environ['DATA_DIR'] + 'data/interim/motion_correction/main/' for i in range(len(input_mmap_file_list)): row = df.iloc[i].copy() motion_correction_output_list.append(motion_correction_output) aligned_movie = movie[timepoints[i]:timepoints[i+1]] file_name = db.create_file_name(2, selected_rows.iloc[i].name) motion_correction_output_aligned = aligned_movie.save(data_dir + file_name + '_els' + '.mmap', order='C') new_output= {'main' : motion_correction_output_aligned } new_dict = eval(row['motion_correction_output']) new_dict.update(new_output) row['motion_correction_output'] = str(new_dict) df = db.append_to_or_merge_with_states_df(df, row) # # Delete the motion corrected movies # for fname in mc.fname_tot_rig: # os.remove(fname) return df
def run_source_extraction(row, parameters, dview, session_wise = False): ''' This is the function for source extraction. Its goal is to take in a .mmap file, perform source extraction on it using cnmf-e and save the cnmf object as a .pkl file. Args: row: pd.DataFrame object The row corresponding to the analysis state to be source extracted. Returns: row: pd.DataFrame object The row corresponding to the source extracted analysis state. ''' step_index = 5 row_local = row.copy() row_local.loc['source_extraction_parameters'] = str(parameters) row_local = db.set_version_analysis('source_extraction',row_local,session_wise) index = row_local.name # Determine input path if parameters['session_wise']: input_mmap_file_path = eval(row_local.loc['alignment_output'])['main'] if parameters['equalization']: input_mmap_file_path =eval(row_local['equalization_output'])['main'] else: input_mmap_file_path = eval(row_local.loc['motion_correction_output'])['main'] if not os.path.isfile(input_mmap_file_path): logging.error('Input file does not exist. Cancelling.') return row_local # Determine output paths file_name = db.create_file_name(step_index, index) if parameters['session_wise']: data_dir = os.environ['DATA_DIR'] + 'data/interim/source_extraction/session_wise/' else: data_dir = os.environ['DATA_DIR'] + 'data/interim/source_extraction/trial_wise/' output_file_path = data_dir + f'main/{file_name}.hdf5' # Create a dictionary with parameters output = { 'main': output_file_path, 'meta':{ 'analysis' : { 'analyst' : os.environ['ANALYST'], 'date' : datetime.datetime.today().strftime("%m-%d-%Y"), 'time' : datetime.datetime.today().strftime("%H:%M:%S"), }, 'duration': {} } } # Load memmory mappable input file if os.path.isfile(input_mmap_file_path): Yr, dims, T = cm.load_memmap(input_mmap_file_path) # logging.debug(f'{index} Loaded movie. dims = {dims}, T = {T}.') images = Yr.T.reshape((T,) + dims, order='F') else: logging.warning(f'{index} .mmap file does not exist. Cancelling') return row_local # SOURCE EXTRACTION # Check if the summary images are already there corr_npy_file_path, pnr_npy_file_path = fm.get_corr_pnr_path(index, gSig_abs = parameters['gSig'][0]) if corr_npy_file_path != None and os.path.isfile(corr_npy_file_path): # Already computed summary images logging.info(f'{index} Already computed summary images') cn_filter = np.load(corr_npy_file_path) pnr = np.load(pnr_npy_file_path) else: # Compute summary images t0 = datetime.datetime.today() logging.info(f'{index} Computing summary images') cn_filter, pnr = cm.summary_images.correlation_pnr(images[::1], gSig = parameters['gSig'][0], swap_dim=False) dt = int((datetime.datetime.today() - t0).seconds/60) # timedelta in minutes output['meta']['duration']['summary_images'] = dt logging.info(f'{index} Computed summary images. dt = {dt} min') # Saving summary images as npy files gSig = parameters['gSig'][0] corr_npy_file_path = data_dir + f'/meta/corr/{db.create_file_name(3, index)}_gSig_{gSig}.npy' pnr_npy_file_path = data_dir + f'/meta/pnr/{db.create_file_name(3, index)}_gSig_{gSig}.npy' with open(corr_npy_file_path, 'wb') as f: np.save(f, cn_filter) with open(pnr_npy_file_path, 'wb') as f: np.save(f, pnr) # Store the paths in the meta dictionary output['meta']['corr'] = {'main': corr_npy_file_path, 'meta': {}} output['meta']['pnr'] = {'main': pnr_npy_file_path, 'meta': {}} # Calculate min, mean, max value for cn_filter and pnr corr_min, corr_mean, corr_max = cn_filter.min(), cn_filter.mean(), cn_filter.max() output['meta']['corr']['meta'] = {'min': corr_min, 'mean': corr_mean, 'max': corr_max} pnr_min, pnr_mean, pnr_max = pnr.min(), pnr.mean(), pnr.max() output['meta']['pnr']['meta'] = {'min': pnr_min, 'mean': pnr_mean, 'max': pnr_max} # If min_corr and min_pnr are specified via a linear equation, calculate # this value if type(parameters['min_corr']) == list: min_corr = parameters['min_corr'][0]*corr_mean + parameters['min_corr'][1] parameters['min_corr'] = min_corr logging.info(f'{index} Automatically setting min_corr = {min_corr}') if type(parameters['min_pnr']) == list: min_pnr = parameters['min_pnr'][0]*pnr_mean + parameters['min_pnr'][1] parameters['min_pnr'] = min_pnr logging.info(f'{index} Automatically setting min_pnr = {min_pnr}') # Set the parameters for caiman opts = params.CNMFParams(params_dict = parameters) # SOURCE EXTRACTION logging.info(f'{index} Performing source extraction') t0 = datetime.datetime.today() n_processes = psutil.cpu_count() logging.info(f'{index} n_processes: {n_processes}') cnm = cnmf.CNMF(n_processes = n_processes, dview = dview, params = opts) cnm.fit(images) cnm.estimates.dims = dims # Store the number of neurons output['meta']['K'] = len(cnm.estimates.C) # Calculate the center of masses cnm.estimates.center = caiman.base.rois.com(cnm.estimates.A, images.shape[1], images.shape[2]) # Save the cnmf object as a hdf5 file logging.info(f'{index} Saving cnmf object') cnm.save(output_file_path) dt = int((datetime.datetime.today() - t0).seconds/60) # timedelta in minutes output['meta']['duration']['source_extraction'] = dt logging.info(f'{index} Source extraction finished. dt = {dt} min') # Write necessary variables in row and return row_local.loc['source_extraction_parameters'] = str(parameters) row_local.loc['source_extraction_output'] = str(output) return row_local
output_cropping = mouse_row['cropping_output'] cropped_file = eval(output_cropping)['main'] m = cm.load(cropped_file) axes[kk, 1].imshow(m[0, :, :], cmap='gray') output_source_extraction = eval( mouse_row['source_extraction_output']) cnm_file_path = output_source_extraction['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) corr_path = output_source_extraction['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) axes[kk, 2].imshow(cn_filter) coordinates = cm.utils.visualization.get_contours( cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [ np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0])) ] axes[kk, 2].plot(*v.T, c='w', linewidth=3) fig_dir = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/interim/cropping/meta/figures/cropping_inicialization/' fig_name = fig_dir + db.create_file_name( 2, mouse_row.name ) + '_corr_' + f'{round(corr_limits[ii],1)}' + '_pnr_' + f'{round(pnr_limits[jj])}' + '.png' figure.savefig(fig_name)
def run_cropper(row, parameters): ''' This function takes in a decoded analysis state and crops it according to specified cropping points. Args: index: tuple The index of the analysis state to be cropped. row: pd.DataFrame object The row corresponding to the analysis state to be cropped. Returns row: pd.DataFrame object The row corresponding to the cropped analysis state. ''' row_local = row.copy() index = row_local.name # Get the input tif file path input_tif_file_path = eval(row_local.loc['decoding_output'])['main'] if index[4] == 2: input_tif_file_path = eval( row_local.loc['decoding_output'])['equalizing_output']['main'] if not os.path.isfile(input_tif_file_path): db.get_expected_file_path('decoding', 'main/', index, '.tif') # Determine output .tif file path step_index = 1 row_local.loc['cropping_parameters'] = str(parameters) row_local = db.set_version_analysis('cropping', row_local) index = row_local.name file_name = db.create_file_name(step_index, index) output_tif_file_path = os.environ[ 'DATA_DIR'] + f"data/interim/cropping/main/{file_name}.tif" # Create a dictionary with the output output = { 'main': output_tif_file_path, 'meta': { 'analysis': { 'analyst': os.environ['ANALYST'], 'date': datetime.datetime.today().strftime("%m-%d-%Y"), 'time': datetime.datetime.today().strftime("%H:%M:%S"), } } } # Spatial copping logging.info(f'{index} Loading movie') m = cm.load(input_tif_file_path) logging.info(f'{index} Loaded movie') [x_, _x, y_, _y] = parameters['cropping_points_spatial'] if parameters['crop_spatial']: logging.info(f'{index} Performing spatial cropping') m = m[:, x_:_x, y_:_y] logging.info(f'{index} Spatial cropping finished') else: logging.info(f'{index} No spatial cropping') # Temporal cropping #if parameters['crop_temporal']: # m, timeline = do_temporal_cropping(m, parameters['cropping_points_temporal']) # The option below is to get a timeline which indicates on which # frames clips are cut out and how long those clips were. # I eventually decided this is not neccesary. The temporal cropping points are enough # to reconstruct this and are more easily saved (namely in the # master file list under 'cropping_parameters') # timeline_pkl_file_path = f'data/interim/cropping/meta/timeline/{file_name}.pkl' # output['meta']['timeline'] = timeline_pkl_file_path # with open(timeline_pkl_file_path,'wb') as f: # pickle.dump(timeline, f) # Save the movie m.save(output_tif_file_path) # Write necessary variables to the trial index and row_local row_local.loc['cropping_parameters'] = str(parameters) row_local.loc['cropping_output'] = str(output) return row_local
def run_component_evaluation(row, parameters, set_version=None, session_wise=False, equalization=False): step_index = 6 row_local = row.copy() row_local.loc['component_evaluation_parameters'] = str(parameters) row_local = db.set_version_analysis('component_evaluation', row_local, session_wise) index = row_local.name motion_correction_output = eval(row_local.loc['motion_correction_output']) if session_wise: motion_correction_output = eval(row_local.loc['alignment_output']) if equalization: motion_correction_output = eval(row_local['equalization_output']) source_extraction_output = eval(row_local.loc['source_extraction_output']) source_extraction_parameters = eval( row_local.loc['source_extraction_parameters']) input_hdf5_file_path = source_extraction_output['main'] input_mmap_file_path = motion_correction_output['main'] data_dir = os.environ[ 'DATA_DIR'] + 'data/interim/component_evaluation/session_wise/' if source_extraction_parameters[ 'session_wise'] else os.environ[ 'DATA_DIR'] + 'data/interim/component_evaluation/trial_wise/' file_name = db.create_file_name(step_index, index) output_file_path = data_dir + f'main/{file_name}.hdf5' # Create a dictionary with parameters output = { 'main': output_file_path, 'meta': { 'analysis': { 'analyst': os.environ['ANALYST'], 'date': datetime.datetime.today().strftime("%m-%d-%Y"), 'time': datetime.datetime.today().strftime("%H:%M:%S"), }, 'duration': {} } } # Load CNMF object cnm = load_CNMF(input_hdf5_file_path) # Load the original movie Yr, dims, T = cm.load_memmap(input_mmap_file_path) images = Yr.T.reshape((T, ) + dims, order='F') # Set the parmeters cnm.params.set('quality', parameters) # Stop the cluster if one exists n_processes = psutil.cpu_count() try: cm.cluster.stop_server() except: pass # Start a new cluster c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes= n_processes, # number of process to use, if you go out of memory try to reduce this one single_thread=False) # Evaluate components cnm.estimates.evaluate_components(images, cnm.params, dview=dview) logging.debug('Number of total components: ', len(cnm.estimates.C)) logging.debug('Number of accepted components: ', len(cnm.estimates.idx_components)) # Stop the cluster dview.terminate() # Save CNMF object cnm.save(output_file_path) # Write necessary variables to the trial index and row row_local.loc['component_evaluation_parameters'] = str(parameters) row_local.loc['component_evaluation_output'] = str(output) return row_local
def plot_multiple_contours_session_wise_evaluated(selected_rows): ## IN DEVELOPMENT!!!!!!! ''' Plots different versions of contour images that change the initialization parameters for source extraction. The idea is to see the impact of different seed selection in the final source extraction result. :param selected_rows: all analysis state selected :return: figure ''' figure, axes = plt.subplots(3, 5, figsize=(50, 30)) for row in range(len(selected_rows)): mouse_row = selected_rows.iloc[row] index = mouse_row.name output = eval(mouse_row.loc['source_extraction_output']) corr_path = output['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) axes[0,row].imshow(cn_filter) axes[1,row].imshow(cn_filter) axes[2,row].imshow(cn_filter) output = eval(mouse_row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[0,row].plot(*v.T, c = 'w', linewidth=3) axes[0,row].set_title('Trial = ' + f'{row}') axes[0,row].set_ylabel('') output = eval(mouse_row.loc['component_evaluation_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) idx = cnm.estimates.idx_components coordinates = cm.utils.visualization.get_contours(cnm.estimates.A[:, idx], np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[1,row].plot(*v.T, c='b', linewidth=3) idx_b = cnm.estimates.idx_components_bad coordinates_b = cm.utils.visualization.get_contours(cnm.estimates.A[:,idx_b], np.shape(cn_filter), 0.2, 'max') for c in coordinates_b: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[2,row].plot(*v.T, c='r', linewidth=3) source_extraction_parameters = eval(mouse_row['source_extraction_parameters']) corr_lim = source_extraction_parameters['min_corr'] pnr_lim = source_extraction_parameters['min_pnr'] component_evaluation_parameters = eval(mouse_row['component_evaluation_parameters']) pcc = component_evaluation_parameters['rval_thr'] SNR = component_evaluation_parameters['min_SNR'] figure.suptitle('Corr = ' + f'{corr_lim}' + 'PNR = ' + f'{pnr_lim}' + 'PCC = ' + f'{pcc}' + 'SNR = ' + f'{SNR}', fontsize=50) fig_dir = 'data/interim/component_evaluation/session_wise/meta/figures/contours/' fig_name = fig_dir + db.create_file_name(3, index)+'_Corr = ' + f'{corr_lim}' + '_PNR = ' + f'{pnr_lim}' + '_PCC = ' + f'{pcc}' + '_SNR = ' + f'{SNR}' +'_.png' figure.savefig(fig_name) return figure
def make_figures(index, row, force=False): # Create file name file_name = db.create_file_name(step_index, index) # Load meta_pkl file output = eval(row.loc['motion_correction_output']) metrics_pkl_file_path = output['meta']['metrics'] with open(metrics_pkl_file_path, 'rb') as f: x = pickle.load(f) # Possible figures figure_names = np.array([ 'rig_template', 'rig_shifts', 'els_template', 'correlations', 'corelations_orig_vs_rig', 'correlations_rig_vs_els', 'correlations_orig_vs_els', 'orig_local_correlations', 'rig_local_correlations', 'els_local_correlations' ]) def figure_flag(i): # This function determines which figures can be made. If they cannot be made, either # an analysis step has not been performed or metrics have to be computed if i == 0: return 'rigid' in x elif i == 1: return 'rigid' in x elif i == 2: return 'non-rigid' in x elif i == 3: return [ 'original' in x and 'correlations' in x['original'], 'rigid' in x and 'correlations' in x['rigid'], 'non-rigid' in x and 'correlations' in x['non-rigid'] ] elif i == 4: return ('original' in x and 'correlations' in x['original']) and ( 'rigid' in x and 'correlations' in x['rigid']) elif i == 5: return ('non-rigid' in x and 'correlations' in x['non-rigid']) and ('rigid' in x and 'correlations' in x['rigid']) elif i == 6: return ('original' in x and 'correlations' in x['original']) and ( 'non-rigid' in x and 'correlations' in x['non-rigid']) elif i == 7: return 'original' in x and 'local_correlations' in x['original'] elif i == 8: return 'rigid' in x and 'local_correlations' in x['rigid'] elif i == 9: return 'non-rigid' in x and 'local_correlations' in x['non-rigid'] def make_figure(i): # This function specifies how each of the figures are layed out if i == 0: plt.imshow(x['rigid']['total_template'], cmap='gray') plt.title('Total template rigid motion correction') elif i == 1: plt.plot(x['rigid']['shifts']) plt.legend(['x shifts', 'y shifts']) plt.xlabel('frames') plt.ylabel('pixels') plt.title('Shifts rigid motion correction') elif i == 2: plt.imshow(x['non-rigid']['total_template'], cmap='gray') plt.title('Total template piecewise-rigid motion correction') elif i == 3: legend = [] for idx, flag in enumerate(figure_flag(i)): if flag: string = ['original', 'rigid', 'non-rigid'][idx] legend += string plt.plot(x[string]['correlations']) plt.xlabel('frames') plt.legend(legend) plt.title('Correlations') elif i == 4: min_cor, max_cor = min(x['original']['correlations'] + x['rigid']['correlations']), max( x['original']['correlations'] + x['rigid']['correlations']) plt.scatter(x['original']['correlations'], x['rigid']['correlations']) plt.xlabel('original') plt.ylabel('rigid') plt.plot([min_cor, max_cor], [min_cor, max_cor], 'r--') plt.title('Original v. rigid correlations') elif i == 5: min_cor, max_cor = min(x['rigid']['correlations'] + x['non-rigid']['correlations']), max( x['rigid']['correlations'] + x['non-rigid']['correlations']) plt.scatter(x['rigid']['correlations'], x['non-rigid']['correlations']) plt.xlabel('rigid') plt.ylabel('non-rigid') plt.plot([min_cor, max_cor], [min_cor, max_cor], 'r--') plt.title('Rigid v. piecewise-rigid correlations') elif i == 6: min_cor, max_cor = min(x['original']['correlations'] + x['non-rigid']['correlations']), max( x['original']['correlations'] + x['non-rigid']['correlations']) plt.scatter(x['original']['correlations'], x['non-rigid']['correlations']) plt.xlabel('original') plt.ylabel('non-rigid') plt.plot([min_cor, max_cor], [min_cor, max_cor], 'r--') plt.title('Original v. piecewise-rigid correlations') elif i == 7: plt.imshow(x['original']['local_correlations']) plt.title('Local correlations original movie') elif i == 8: plt.imshow(x['rigid']['local_correlations']) plt.title('Local correlations rigid motion correction') elif i == 9: plt.imshow(x['non-rigid']['local_correlations']) plt.title('Local correlations piecewise-rigid motion correction') for i in range(0, len(figure_names)): if (figure_flag(i) if i != 3 else (True in figure_flag(i))): plt.figure() make_figure(i) plt.savefig( f'data/interim/motion_correction/meta/figures/{file_name}_{figure_names[i]}.png' ) plt.close() return
for kk in range(1, len(source_extraction_v_array)): for ll in range(1, len(component_evaluation_v_array)): list1.append(number_cell[kk - 1][ll - 1]) list2.append(1 - fraction[kk - 1][ll - 1]) list3.append(number_cell5[kk - 1][ll - 1]) list4.append(1 - fraction5[kk - 1][ll - 1]) product = np.arange(0, len(list1)) false_positive = np.zeros((len(list1))) false_positive_next = np.zeros((len(list1))) for ii in range(len(product) - 1): product[ii] = list3[ii] * list2[ii] false_positive[ii] = list4[ii] false_positive_next[ii] = list4[ii + 1] file_name_number = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/interim/component_evaluation/session_wise/meta/metrics/' + db.create_file_name( 5, mouse_row.name) + '_number' np.save(file_name_number, number_cell5) file_name_fraction = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/interim/component_evaluation/session_wise/meta/metrics/' + db.create_file_name( 5, mouse_row.name) + '_fraction' np.save(file_name_fraction, fraction5) #%% Working with matching!!!!!! one_version = 13 selected_rows = db.select(states_df, 'source_extraction', 56165, cropping_v=1, motion_correction_v=1, source_extraction_v=one_version)
def create_video(row, time_cropping, session_wise = False): ''' This fuction creates a complete video with raw movie (motion corrected), source extracted cells and source extraction + background. :param row: pandas dataframe containing the desired processing information to create the video. It can use the session_wise or trial_wise video. :return: ''' if session_wise: input_mmap_file_path = eval(row.loc['alignment_output'])['main'] else: input_mmap_file_path = eval(row.loc['motion_correction_output'])['main'] #load the mmap file Yr, dims, T = cm.load_memmap(input_mmap_file_path) logging.debug(f'{row.name} Loaded movie. dims = {dims}, T = {T}.') #create a caiman movie with the mmap file images = Yr.T.reshape((T,) + dims, order='F') images = cm.movie(images) #load source extraction result output = eval(row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) #estimate the background from the extraction W, b0 = cm.source_extraction.cnmf.initialization.compute_W(Yr, cnm.estimates.A.toarray(), cnm.estimates.C, cnm.estimates.dims, 1.4 * 5, ssub=2) cnm.estimates.W = W cnm.estimates.b0 = b0 # this part could be use with the lastest caiman version # movie_dir = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/processed/movies/' # file_name = db.create_file_name(5,row.name) # cnm.estimates.play_movie(cnm.estimates, images, movie_name= movie_dir + file_name + '.avi') frame_range = slice(None, None, None) # create a movie with the model : estimated A and C matrix Y_rec = cnm.estimates.A.dot(cnm.estimates.C[:, frame_range]) Y_rec = Y_rec.reshape(dims + (-1,), order='F') Y_rec = Y_rec.transpose([2, 0, 1]) # convert the variable to a caiman movie type Y_rec = cm.movie(Y_rec) ## this part of the function is a copy from a caiman version ssub_B = int(round(np.sqrt(np.prod(dims) / W.shape[0]))) B = images[frame_range].reshape((-1, np.prod(dims)), order='F').T - \ cnm.estimates.A.dot(cnm.estimates.C[:, frame_range]) if ssub_B == 1: B = b0[:, None] + W.dot(B - b0[:, None]) else: B = b0[:, None] + (np.repeat(np.repeat(W.dot( downscale(B.reshape(dims + (B.shape[-1],), order='F'), (ssub_B, ssub_B, 1)).reshape((-1, B.shape[-1]), order='F') - downscale(b0.reshape(dims, order='F'), (ssub_B, ssub_B)).reshape((-1, 1), order='F')) .reshape( ((dims[0] - 1) // ssub_B + 1, (dims[1] - 1) // ssub_B + 1, -1), order='F'), ssub_B, 0), ssub_B, 1)[:dims[0], :dims[1]].reshape( (-1, B.shape[-1]), order='F')) B = B.reshape(dims + (-1,), order='F').transpose([2, 0, 1]) Y_rec_2 = Y_rec + B Y_res = images[frame_range] - Y_rec - B images_np = np.zeros((time_cropping[1]-time_cropping[0],images.shape[1],images.shape[2])) images_np = images[time_cropping[0]:time_cropping[1],:,:] images_np = images_np / np.max(images_np) images_np = cm.movie(images_np) Y_rec_np = np.zeros((time_cropping[1]-time_cropping[0],images.shape[1],images.shape[2])) Y_rec_np = Y_rec[time_cropping[0]:time_cropping[1],:,:] Y_rec_np = Y_rec_np / np.max(Y_rec_np) Y_rec_np = cm.movie(Y_rec_np) Y_res_np = np.zeros((time_cropping[1]-time_cropping[0],images.shape[1],images.shape[2])) Y_res_np = Y_res[time_cropping[0]:time_cropping[1],:,:] Y_res_np = Y_res_np / np.max(Y_res_np) Y_res_np = cm.movie(Y_res_np) B_np = np.zeros((time_cropping[1]-time_cropping[0],images.shape[1],images.shape[2])) B_np = B[time_cropping[0]:time_cropping[1],:,:] B_np = B_np / np.max(B_np) B_np = cm.movie(B_np) mov1 = cm.concatenate((images_np, Y_rec_np), axis=2) mov2 = cm.concatenate((B_np, Y_res_np), axis=2) mov = cm.concatenate((mov1, mov2), axis=1) figure_path = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/interim/movies/' figure_name = db.create_file_name(5,row.name) #mov.save(figure_path+figure_name+'.tif') mov.save(figure_path+figure_name+'_'+f'{time_cropping[0]}' + '_' + f'{time_cropping[1]}'+'.tif') return
def run_motion_correction(row, parameters, dview): ''' This is the function for motion correction. Its goal is to take in a decoded and cropped .tif file, perform motion correction, and save the result as a .mmap file. This function is only runnable on the cn76 server because it requires parralel processing. Args: row: pd.DataFrame object The row corresponding to the analysis state to be motion corrected. parametrs : motion_correction_parameters dview: cluster Returns: row: pd.DataFrame object The row corresponding to the motion corrected analysis state. ''' step_index = 2 row_local = row.copy() index = row_local.name # Forcing parameters if not parameters['pw_rigid']: parameters['save_movie_rig'] = True # Get input file input_tif_file_path = eval(row_local.loc['cropping_output'])['main'] if not os.path.isfile(input_tif_file_path): input_tif_file_path = db.get_expected_file_path( 'cropping', 'main/', index, 'tif') if not os.path.isfile(input_tif_file_path): logging.error( 'Cropping file not found. Cancelling motion correction.') return row_local row_local.loc['motion_correction_parameters'] = str(parameters) row_local = db.set_version_analysis('motion_correction', row_local) index = row_local.name # Get output file paths data_dir = os.environ['DATA_DIR'] + 'data/interim/motion_correction/' file_name = db.create_file_name(step_index, index) output_meta_pkl_file_path = data_dir + f'meta/metrics/{file_name}.pkl' # Create a dictionary with the output output = { 'meta': { 'analysis': { 'analyst': os.environ['ANALYST'], 'date': datetime.datetime.today().strftime("%m-%d-%Y"), 'time': datetime.datetime.today().strftime("%H:%M:%S") }, 'metrics': { 'other': output_meta_pkl_file_path } } } # Calculate movie minimum to subtract from movie min_mov = np.min(cm.load(input_tif_file_path)) # Apply the parameters to the CaImAn algorithm caiman_parameters = parameters.copy() caiman_parameters['min_mov'] = min_mov opts = params.CNMFParams(params_dict=caiman_parameters) # Rigid motion correction (in both cases) logging.info(f'{index} Performing rigid motion correction') t0 = datetime.datetime.today() # Create a MotionCorrect object mc = MotionCorrect([input_tif_file_path], dview=dview, **opts.get_group('motion')) # Perform rigid motion correction mc.motion_correct_rigid(save_movie=parameters['save_movie_rig'], template=None) dt = int( (datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes logging.info(f'{index} Rigid motion correction finished. dt = {dt} min') # Obtain template, rigid shifts and border pixels total_template_rig = mc.total_template_rig shifts_rig = mc.shifts_rig # Save template, rigid shifts and border pixels in a dictionary meta_pkl_dict = { 'rigid': { 'template': total_template_rig, 'shifts': shifts_rig, } } output['meta']['duration'] = {'rigid': dt} output['meta']['cropping_points'] = [0, 0, 0, 0] if parameters['save_movie_rig']: # Load the movie saved by CaImAn, which is in the wrong # directory and is not yet cropped logging.info(f'{index} Loading rigid movie for cropping') m_rig = cm.load(mc.fname_tot_rig[0]) logging.info(f'{index} Loaded rigid movie for cropping') # Get the cropping points determined by the maximal rigid shifts x_, _x, y_, _y = get_crop_from_rigid_shifts(shifts_rig) # Crop the movie logging.info( f'{index} Cropping and saving rigid movie with cropping points: [x_, _x, y_, _y] = {[x_, _x, y_, _y]}' ) m_rig = m_rig.crop(x_, _x, y_, _y, 0, 0) meta_pkl_dict['rigid']['cropping_points'] = [x_, _x, y_, _y] output['meta']['cropping_points'] = [x_, _x, y_, _y] # Save the movie rig_role = 'alternate' if parameters['pw_rigid'] else 'main' fname_tot_rig = m_rig.save(data_dir + rig_role + '/' + file_name + '_rig' + '.mmap', order='C') logging.info( f'{index} Cropped and saved rigid movie as {fname_tot_rig}') # Store the total path in output output[rig_role] = fname_tot_rig # Remove the remaining non-cropped movie os.remove(mc.fname_tot_rig[0]) # If specified in the parameters, apply piecewise-rigid motion correction if parameters['pw_rigid']: logging.info(f'{index} Performing piecewise-rigid motion correction') t0 = datetime.datetime.today() # Perform non-rigid (piecewise rigid) motion correction. Use the rigid result as a template. mc.motion_correct_pwrigid(save_movie=True, template=total_template_rig) # Obtain template and filename total_template_els = mc.total_template_els fname_tot_els = mc.fname_tot_els[0] dt = int((datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes meta_pkl_dict['pw_rigid'] = { 'template': total_template_els, 'x_shifts': mc.x_shifts_els, 'y_shifts': mc. y_shifts_els # removed them initially because they take up space probably } output['meta']['duration']['pw_rigid'] = dt logging.info( f'{index} Piecewise-rigid motion correction finished. dt = {dt} min' ) # Load the movie saved by CaImAn, which is in the wrong # directory and is not yet cropped logging.info(f'{index} Loading pw-rigid movie for cropping') m_els = cm.load(fname_tot_els) logging.info(f'{index} Loaded pw-rigid movie for cropping') # Get the cropping points determined by the maximal rigid shifts x_, _x, y_, _y = get_crop_from_pw_rigid_shifts( np.array(mc.x_shifts_els), np.array(mc.y_shifts_els)) # Crop the movie logging.info( f'{index} Cropping and saving pw-rigid movie with cropping points: [x_, _x, y_, _y] = {[x_, _x, y_, _y]}' ) m_els = m_els.crop(x_, _x, y_, _y, 0, 0) meta_pkl_dict['pw_rigid']['cropping_points'] = [x_, _x, y_, _y] output['meta']['cropping_points'] = [x_, _x, y_, _y] # Save the movie fname_tot_els = m_els.save(data_dir + 'main/' + file_name + '_els' + '.mmap', order='C') logging.info( f'{index} Cropped and saved rigid movie as {fname_tot_els}') # Remove the remaining non-cropped movie os.remove(mc.fname_tot_els[0]) # Store the total path in output output['main'] = fname_tot_els output['meta']['cropping_points'] = [x_, _x, y_, _y] # Write meta results dictionary to the pkl file pkl_file = open(output_meta_pkl_file_path, 'wb') pickle.dump(meta_pkl_dict, pkl_file) pkl_file.close() # Write necessary variables to the trial index and row row_local.loc['motion_correction_output'] = str(output) row_local.loc['motion_correction_parameters'] = str(parameters) return row_local
def run_equalizer(selected_rows, states_df, parameters, session_wise=False): ''' This function is meant to help with differences in contrast in different trials and session, to equalize general brightness or reduce photobleaching. It corrects the video and saves them in the corrected version. It can be run with the already aligned videos or trial by trial. for trial by trial, a template is required. params: selected_rows: pd.DataFrame -> A dataframe containing the analysis states you want to have equalized params: states_df: pd.DataFrame -> A dataframe containing all the analysis data base params: parameters: dict -> contains parameters concerning equalization returns : None ''' step_index = 4 # Sort the dataframe correctly df = selected_rows.sort_values(by=paths.multi_index_structure) # Determine the output path output_tif_file_path = os.environ[ 'DATA_DIR'] + f'data/interim/equalizer/main/' mouse, session, init_trial, *r = df.iloc[0].name #histogram_name = f'mouse_{mouse}_session_{session}_init_trial_{init_trial}' #output_steps_file_path = f'data/interim/equalizer/meta/figures/histograms/'+histogram_name try: df.reset_index()[['session', 'trial', 'is_rest' ]].set_index(['session', 'trial', 'is_rest'], verify_integrity=True) except ValueError: logging.error( 'You passed multiple of the same trial in the dataframe df') return df #creates an output dictionary for the data base output = { 'main': {}, 'meta': { 'analysis': { 'analyst': os.environ['ANALYST'], 'date': datetime.datetime.today().strftime("%m-%d-%Y"), 'time': datetime.datetime.today().strftime("%H:%M:%S") }, 'duration': {} } } if session_wise: row_local = df.iloc[0] input_tif_file_list = eval(row_local['alignment_output'])['main'] movie_original = cm.load( input_tif_file_list) # load video as 3d array already concatenated if parameters['make_template_from_trial'] == 0: movie_equalized = do_equalization(movie_original) else: movie_equalized = np.empty_like(movie_original) source = movie_original[0:100, :, :] # equalize all the videos loads in m_list_reshape with the histogram of source for j in range(int(movie_original.shape[0] / 100)): want_to_equalize = movie_original[j * 100:(j + 1) * 100, :, :] movie_equalized[j * 100:(j + 1) * 100, :, :] = do_equalization_from_template( reference=want_to_equalize, source=source) #Save the movie index = row_local.name new_index = db.replace_at_index1(index, 4 + 4, 2) ## version 2 is for session wise row_local.name = new_index equalized_path = movie_equalized.save( output_tif_file_path + db.create_file_name(4, row_local.name) + '.mmap', order='C') output['main'] = equalized_path #auxiliar = eval(row_local.loc['alignment_output']) #auxiliar.update({'equalizing_output' : output}) # row_local.loc['alignment_output'] = str(auxiliar) row_local.loc['equalization_output'] = output states_df = db.append_to_or_merge_with_states_df(states_df, row_local) else: # Get necessary parameters and create a list with the paths to the relevant files decoding_output_list = [] input_tif_file_list = [] trial_index_list = [] for idx, row in df.iterrows(): decoding_output = eval(row.loc['decoding_output']) decoding_output_list.append(decoding_output) input_tif_file_list.append(decoding_output['main']) trial_index_list.append(db.get_trial_name(idx[2], idx[3])) # this was something for ploting while testing, can be removed #colors = [] #for i in range(len(df)): # colors.append('#%06X' % randint(0, 0xFFFFFF)) #load the videos as np.array to be able to manipulate them m_list = [] legend = [] shape_list = [] h_step = parameters['histogram_step'] for i in range(len(input_tif_file_list)): im = io.imread(input_tif_file_list[i]) #load video as 3d array m_list.append(im) # and adds all the videos to a list shape_list.append( im.shape ) # list of sizes to cut the videos in time for making all of them having the same length #legend.append('trial = ' + f'{df.iloc[i].name[2]}') min_shape = min(shape_list) new_shape = (100 * int(min_shape[0] / 100), min_shape[1], min_shape[2] ) # new videos shape m_list_reshape = [] m_list_equalized = [] source = m_list[0][0:100, :, :] #equalize all the videos loaded in m_list_reshape with the histogram of source for i in range(len(input_tif_file_list)): video = m_list[i] if parameters['make_template_from_trial'] == 0: equalized_video = do_equalization(video) else: m_list_reshape.append(video[:new_shape[0], :, :]) equalized_video = np.empty_like(video[:new_shape[0], :, :]) for j in range(int(min_shape[0] / 100)): want_to_equalize = m_list_reshape[i][j * 100:(j + 1) * 100, :, :] equalized_video[j * 100:(j + 1) * 100, :, :] = do_equalization_from_template( reference=want_to_equalize, source=source) m_list_equalized.append(equalized_video) #convert the 3d np.array to a caiman movie and save it as a tif file, so it can be read by motion correction script. for i in range(len(input_tif_file_list)): # Save the movie row_local = df.iloc[i] movie_original = cm.movie(m_list_reshape[i]) movie_equalized = cm.movie(m_list_equalized[i]) # Write necessary variables to the trial index and row_local index = row_local.name new_index = db.replace_at_index1(index, 4 + 0, 1) ## version 1 is for trial wise row_local.name = new_index output['main'] = output_tif_file_path + db.create_file_name( 4, row_local.name) + '.tif' #auxiliar = eval(row_local.loc['decoding_output']) #auxiliar.update({'equalizing_output' : output}) #row_local.loc['decoding_output'] = str(auxiliar) row_local.loc['equalization_output'] = output movie_equalized.save(output_tif_file_path + db.create_file_name(4, row_local.name) + '.tif') states_df = db.append_to_or_merge_with_states_df( states_df, row_local) db.save_analysis_states_database(states_df, paths.analysis_states_database_path, paths.backup_path) return
def run_equalizer(states_df, parameters): ''' This function is meant to help with differences in different trials and session, to equalize general brightness or reduce photobleaching. It corrects the video and saves them in the corrected version. params df: pd.DataFrame -> A dataframe containing the analysis states you want to have aligned. params parameters: dict -> contains parameters concerning equalization returns df: pd.DataFrame -> A dataframe containing the aligned analysis states. ''' # Sort the dataframe correctly df = states_df.sort_values(by=paths.multi_index_structure) # Determine the output path output_mmap_file_path = f'data/interim/equalizer/main/' mouse, session, init_trial, *r = df.iloc[0].name histogram_name = f'mouse_{mouse}_session_{session}_init_trial_{init_trial}' output_steps_file_path = f'data/interim/equalizer/meta/figures/histograms/' + histogram_name try: df.reset_index()[['trial', 'is_rest']].set_index(['trial', 'is_rest'], verify_integrity=True) except ValueError: logging.error( 'You passed multiple of the same trial in the dataframe df') return df output = { 'meta': { 'analysis': { 'analyst': os.environ['ANALYST'], 'date': datetime.datetime.today().strftime("%m-%d-%Y"), 'time': datetime.datetime.today().strftime("%H:%M:%S") }, 'duration': {} } } # Get necessary parameters decoding_output_list = [] input_tif_file_list = [] trial_index_list = [] for idx, row in df.iterrows(): decoding_output = eval(row.loc['decoding_output']) decoding_output_list.append(decoding_output) input_tif_file_list.append(decoding_output['main']) trial_index_list.append(db.get_trial_name(idx[2], idx[3])) colors = [] for i in range(len(df)): colors.append('#%06X' % randint(0, 0xFFFFFF)) m_list = [] m_reshape_list = [] legend = [] min_list = [] max_list = [] h_step = parameters['histogram_step'] fig, ax = plt.subplots(1, 2) for i in range(len(input_tif_file_list)): im = io.imread(input_tif_file_list[i]) #m_list.append(im) reshape_m = np.reshape(im, (im.shape[0] * im.shape[1] * im.shape[2])) m_reshape_list.append(reshape_m) legend.append('trial = ' + f'{df.iloc[i].name[2]}') total_min = 200 total_max = 1500 cdf_list = [] for i in range(len(input_tif_file_list)): hist, bins = np.histogram( m_reshape_list[i], bins=np.linspace(total_min, total_max, (total_max - total_min) / h_step), density=True) cdf = np.cumsum(hist) * h_step cdf_list.append(cdf) ax[0].plot(bins[0:-1], hist, color=colors[i]) ax[1].plot(bins[0:-1], cdf, color=colors[i]) ax[0].set_xlabel('Pixel Intensity') ax[1].set_xlabel('Pixel Intensity') ax[0].set_ylabel('Density') ax[1].set_ylabel('CDF') ax[0].legend((legend)) fig.savefig(output_steps_file_path + '.png') # Concatenate them using the concat function m_concat = cm.concatenate(m_list, axis=0) data_dir = 'data/interim/alignment/main' file_name = db.create_file_name(step_index, index) fname = m_concat.save(data_dir + '/' + file_name + '_pw_rig' + '.mmap', order='C') cdf_m = np.ma.masked_equal(cdf, 0) cdf_m = (cdf_m - cdf_m.min()) * 255 / (cdf_m.max() - cdf_m.min()) cdf = np.ma.filled(cdf_m, 0).astype('uint8') #meta_pkl_dict['pw_rigid']['cropping_points'] = [x_, _x, y_, _y] #output['meta']['cropping_points'] = [x_, _x, y_, _y] # Save the movie #fname_tot_els = m_els.save(data_dir + 'main/' + file_name + '_els' + '.mmap', order='C') #logging.info(f'{index} Cropped and saved rigid movie as {fname_tot_els}') # MOTION CORRECTING EACH INDIVIDUAL MOVIE WITH RESPECT TO A TEMPLATE MADE OF THE FIRST MOVIE logging.info( f'{alignment_index} Performing motion correction on all movies with respect to a template made of \ the first movie.') t0 = datetime.datetime.today() # Create a template of the first movie template_index = trial_index_list.index( parameters['make_template_from_trial']) m0 = cm.load(input_mmap_file_list[template_index]) [x1, x2, y1, y2] = motion_correction_output_list[template_index]['meta'][ 'cropping_points'] m0 = m0.crop(new_x1 - x1, new_x2 - x2, new_y1 - y1, new_y2 - y2, 0, 0) m0_filt = cm.movie( np.array([ high_pass_filter_space(m_, parameters['gSig_filt']) for m_ in m0 ])) template0 = cm.motion_correction.bin_median( m0_filt.motion_correct( 5, 5, template=None)[0]) # may be improved in the future # Setting the parameters opts = params.CNMFParams(params_dict=parameters) # Create a motion correction object mc = MotionCorrect(fname, dview=dview, **opts.get_group('motion')) # Perform non-rigid motion correction mc.motion_correct(template=template0, save_movie=True) # Cropping borders x_ = math.ceil( abs(np.array(mc.shifts_rig)[:, 1].max() ) if np.array(mc.shifts_rig)[:, 1].max() > 0 else 0) _x = math.ceil( abs(np.array(mc.shifts_rig)[:, 1].min() ) if np.array(mc.shifts_rig)[:, 1].min() < 0 else 0) y_ = math.ceil( abs(np.array(mc.shifts_rig)[:, 0].max() ) if np.array(mc.shifts_rig)[:, 0].max() > 0 else 0) _y = math.ceil( abs(np.array(mc.shifts_rig)[:, 0].min() ) if np.array(mc.shifts_rig)[:, 0].min() < 0 else 0) dt = int( (datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes output['meta']['duration']['motion_correction'] = dt logging.info( f'{alignment_index} Performed motion correction. dt = {dt} min.') # Create a timeline and store it timeline = [[trial_index_list[0], 0]] for i in range(1, len(m_list)): m = m_list[i] timeline.append([trial_index_list[i], timeline[i - 1][1] + m.shape[0]]) # timeline_pkl_file_path = f'data/interim/alignment/meta/timeline/{file_name}.pkl' # with open(timeline_pkl_file_path,'wb') as f: # pickle.dump(timeline,f) # output['meta']['timeline'] = timeline_pkl_file_path output['meta']['timeline'] = timeline # Save the concatenated movie output_mmap_file_path_tot = m_concat.save(output_mmap_file_path, order='C') output['main'] = output_mmap_file_path_tot # # Delete the motion corrected movies # for fname in mc.fname_tot_rig: # os.remove(fname) dt = int( (datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes output['meta']['duration']['concatenation'] = dt logging.info(f'{alignment_index} Performed concatenation. dt = {dt} min.') for idx, row in df.iterrows(): df.loc[idx, 'alignment_output'] = str(output) df.loc[idx, 'alignment_parameters'] = str(parameters) return df