#SGPMD-NMF trace_pmd = io.imread('/home/nel/Code/volpy_test/invivo-imaging/test_data/09282017Fish1-1/output/temporal_traces.tif')[1] trace_pmd = np.concatenate((np.array([0]*100), trace_pmd))[:len(frame_timing)] trace_pmd = signal_filter(trace_pmd, freq=15, fr=400) trace_pmd = trace_pmd / np.max(trace_pmd) plt.plot(frame_timing, trace_pmd, label='pmd',color='red') spikes_pmd = scipy.signal.find_peaks(trace_pmd, 0.45, distance=5)[0] plt.plot(frame_timing[spikes_pmd], np.max(ephys) * 1.3 * np.ones(spikes_pmd.shape), color='red', marker='.', fillstyle='none', linestyle='none') plt.legend() """ #%% import caiman as cm from caiman.base.rois import nf_read_roi_zip masks = nf_read_roi_zip(os.path.join(root_dir, 'mask.zip'), dims=(44, 128)) masks_m = masks[0].reshape(-1, order='F') m = cm.load('/home/nel/data/voltage_data/simul_electr/johannes/09282017Fish1-1/memmap__d1_44_d2_128_d3_1_order_C_frames_37950_.mmap') mm = m.reshape((m.shape[0], -1), order='F') trace_mean = (mm[:, masks_m>0]).mean(1) trace_mean = signal_filter(-trace_mean[np.newaxis, :], freq=1/3, fr=300)[0] trace_mean = signal_filter(trace_mean[np.newaxis, :], freq=10, fr=300)[0] trace_mean = trace_mean / trace_mean.max() plt.plot(trace_mean) spikes_mean = scipy.signal.find_peaks(trace_mean, 0.3, distance=5)[0] spikes_mean = np.delete(spikes_mean, np.where(spikes_mean>len(frame_timing))).astype(np.int32) #%% import caiman as cm from caiman.base.rois import nf_read_roi_zip masks = nf_read_roi_zip(os.path.join(root_dir, 'mask.zip'), dims=(32, 64))
#%% for folder_out in folders_out: projection_img_median = folder_out + '/projections/median_projection.tif' projection_img_correlation = folder_out + '/projections/correlation_image.tif' folder_in = folder_out + '/regions' performance_all = dict() fls = list(glob.glob(folder_in + '/*_nd.zip')) consensus_counter = dict() fl1 = fls[0] for fl2 in fls[1:]: print([fl1, fl2]) Cn = cm.load(projection_img_correlation) shape = Cn.shape roi_1, names_1 = nf_read_roi_zip(fl1, shape, return_names=True) roi_2, names_2 = nf_read_roi_zip(fl2, shape, return_names=True) # pl.figure() # pl.imshow(np.sum(roi_1,0),cmap = 'gray',vmax=2,alpha=.5) # pl.imshow(np.sum(roi_2,0),cmap = 'hot',vmax=2,alpha=.5) lab1, lab2 = fl1.split('/')[-1][:-4], fl2.split('/')[-1][:-4] # pl.figure(figsize=(15,10)) tp_gt, tp_comp, fn_gt, fp_comp, performance = cm.base.rois.nf_match_neurons_in_binary_masks( roi_1, roi_2, thresh_cost=.7, min_dist=10, print_assignment=False, plot_results=False, Cn=Cn, labels=[lab1, lab2])
'regions/sonia_all_regions.zip', 'regions/ben_all_regions.zip' ] results = dict() for count, par in enumerate(params): print( os.path.join('/mnt/ceph/neuro/labeling/', par[0], 'projections/correlation_image.tif')) c_img = cm.load( os.path.join('/mnt/ceph/neuro/labeling/', par[0], 'projections/correlation_image.tif')) result = dict() for region_pairs in itertools.combinations(label_name, 2): print(region_pairs) try: roi_nat = nf_read_roi_zip( os.path.join('/mnt/ceph/neuro/labeling/', par[0], region_pairs[0]), c_img.shape) roi_lin = nf_read_roi_zip( os.path.join('/mnt/ceph/neuro/labeling/', par[0], region_pairs[1]), c_img.shape) print(roi_nat.shape) print(roi_lin.shape) # roi_nat = nf_read_roi_zip(os.path.join('/mnt/ceph/neuro/labeling/',par[0],'regions/natalia_all_regions.zip'),c_img.shape) # roi_lin = nf_read_roi_zip(os.path.join('/mnt/ceph/neuro/labeling/',par[0],'regions/lindsey_all_regions.zip'),c_img.shape) except: print('******************** PROBLEMS WITH ' + par[0]) continue # # pl.subplot(3,2,count+1) # c_img[np.isnan(c_img )] = 0
#!/usr/bin/env python # -*- coding: utf-8 -*- """ Created on Fri Jul 14 09:42:20 2017 @author: agiovann """ from caiman.base.rois import nf_read_roi_zip # data transformation because of motion correction mismatch in shape #%% sue k37 new_templ = cm.load('projections/median_projection.tif') regions = nf_read_roi_zip('regions/joined_consensus_active_regions.zip', new_templ.shape) new_templ = np.pad(new_templ.T, ((7, 8), (4, 3)), mode='constant', constant_values=new_templ.mean()) regions = np.pad(regions.transpose(0, 2, 1), ((0, 0), (7, 8), (4, 3)), mode='constant') np.save('regions/joined_consensus_active_regions', regions) np.save('projections/median_projection', new_templ) #%% if no change corr_img = cm.load('projections/correlation_image.tif') new_templ = cm.load('projections/median_projection.tif') regions = nf_read_roi_zip('regions/joined_consensus_active_regions.zip', new_templ.shape) pl.imshow(new_templ / np.nanmax(new_templ), cmap='gray', vmin=0.01, vmax=.3) #pl.imshow(corr_img/np.nanmax(corr_img),cmap = 'gray',vmax = .25)
pl.imshow(A_on_thr.sum(-1).reshape(dims_on, order='F')) #%% roi_cons = np.load( '/mnt/ceph/neuro/labeling/neurofinder.02.00/regions/joined_consensus_active_regions.npy' ) print(roi_cons.shape) pl.imshow(roi_cons.sum(0)) #%% roi_cons = np.load( '/mnt/ceph/neuro/labeling/neurofinder.03.00.test/regions/joined_consensus_active_regions.npy' ) print(roi_cons.shape) pl.imshow(roi_cons.sum(0)) #%% roi_cons = nf_read_roi_zip( '/mnt/ceph/neuro/labeling/neurofinder.02.00/regions/ben_active_regions_nd_natalia_active_regions_nd__sonia_active_regions_nd__lindsey_active_regions_nd_matches.zip', Cn.shape) print(roi_cons.shape) #%% SUE roi_cons = nf_read_roi_zip( '/mnt/ceph/neuro/labeling/k53_20160530/regions/ben_active_regions_nd_natalia_active_regions_nd__sonia_active_regions_nd__lindsey_active_regions_nd_matches.zip', Cn.shape) print(roi_cons.shape) #%% roi_cons = np.load( '/mnt/ceph/neuro/labeling/J115_2015-12-09_L01_ELS/regions/joined_consensus_active_regions.npy' ) #roi_cons = np.load('/mnt/ceph/neuro/labeling/k53_20160530/regions/joined_consensus_active_regions.npy') print(roi_cons.shape) #%% roi_cons = nf_read_roi_zip(
v = np.load(os.path.join(path, files), allow_pickle=True).item() v_temporal = v['ts'][v_idx][:len(frame_timing)] v_temporal = v_temporal - np.mean(v_temporal) v_spikes = v['spikes'][v_idx].copy() v_spikes = np.delete(v_spikes, np.where(v_spikes > len(frame_timing))).astype(np.int32) v_spatial = v['weights'][v_idx].copy() v_temporal = v_temporal / np.max(v_temporal) plt.figure() plt.plot(v_temporal) plt.figure() plt.imshow(v_spatial) # Mean ROI result folder = os.path.join(ROOT_FOLDER, name) masks = nf_read_roi_zip(os.path.join(folder, 'mask.zip'), dims=dims) masks_m = masks[0].reshape(-1, order='F') mov_path = os.listdir(os.path.join(folder, 'volpy')) mov_path = [p for p in mov_path if 'F_frames' in p][0] m = cm.load(os.path.join(folder, 'volpy', mov_path)) mm = m.reshape((m.shape[0], -1), order='F') m_temporal = (mm[:, masks_m > 0]).mean(1) m_temporal = signal_filter(-m_temporal[np.newaxis, :], freq=15, fr=fr)[0] m_temporal = m_temporal / m_temporal.max() m_spikes = scipy.signal.find_peaks(m_temporal, m_height, distance=int(fr * 0.01))[0] m_spikes = np.delete(m_spikes, np.where(m_spikes > len(frame_timing))).astype(np.int32) plt.plot(m_temporal)
'/mnt/ceph/neuro/labeling/neurofinder.04.00.test'] foldernames = ['/mnt/ceph/neuro/labeling/FINAL_NO_USED_FOR_CONSENSUS/neurofinder.01.01', '/mnt/ceph/neuro/labeling/FINAL_NO_USED_FOR_CONSENSUS/packer.001', '/mnt/ceph/neuro/labeling/FINAL_NO_USED_FOR_CONSENSUS/Yi.data.001', '/mnt/ceph/neuro/labeling/FINAL_NO_USED_FOR_CONSENSUS/yuste.Single_150u', '/mnt/ceph/neuro/labeling/FINAL_NO_USED_FOR_CONSENSUS/Jan-AMG1_exp2_new_001'] #%% import glob from caiman.base.rois import detect_duplicates, nf_merge_roi_zip, nf_read_roi_zip from shutil import copyfile #%% for fldname in foldernames: current_folder = os.path.join( '/mnt/ceph/neuro/labeling', fldname, 'regions') img_shape = cm.load(os.path.join('/mnt/ceph/neuro/labeling', fldname, 'projections/correlation_image.tif')).shape filenames = glob.glob(os.path.join(current_folder, '*active*regions.zip')) for flname in filenames: ind_dup, ind_keep = detect_duplicates(flname, 0.25, FOV=img_shape) rois = nf_read_roi_zip(flname, img_shape) new_fname = flname[:-4] + '_nd.zip' print(flname) if not ind_dup: copyfile(flname, new_fname) else: nf_merge_roi_zip([flname], [ind_dup], flname[:-4] + '_copy') nf_merge_roi_zip([flname], [ind_keep], new_fname[:-4]) print('FOUND!!')
print(np.mean(m, axis=(0,1))) a.append(np.mean(m, axis=(0,1))) #%% Save mask from .zip to json from caiman.base.rois import nf_read_roi_zip from caiman.base.rois import nf_masks_to_json img_dir = '/home/nel/data/voltage_data/volpy_paper/img' zip_dir = '/home/nel/data/voltage_data/volpy_paper/zip' mask_dir = '/home/nel/data/voltage_data/volpy_paper/mask' fls = os.listdir(img_dir) for fi in fls: img_path = os.path.join(img_dir, fi) dims = cv2.imread(img_path).shape[:2] zip_path = os.path.join(zip_dir, fi[:-4]+'.zip') masks = nf_read_roi_zip(zip_path, dims=dims) mask_path = os.path.join(mask_dir, fi[:-4]+'.json') nf_masks_to_json(masks, os.path.join(mask_path)) #%% Zip to json dr = '/home/nel/data/voltage_data/volpy_paper/img' dr_mask = '/home/nel/data/voltage_data/volpy_paper/mask' files = sorted(os.listdir(dr)) files = [file for file in files if '.npz' in file] for file in files: m = np.load(os.path.join(dr, file), allow_pickle=True)['arr_0'] dims = m.shape[:2] mask = nf_read_roi_zip(os.path.join(dr_mask, file[:-4]+'.zip'), dims=dims)
par[0], 'projections/correlation_image.tif')) c_img = cm.load(os.path.join('/mnt/ceph/neuro/labeling/', par[0], 'projections/correlation_image.tif')) result = dict() if compare_code == 0: iterlabels = label_name elif compare_code == 1: iterlabels = itertools.combinations(label_name, 2) else: raise Exception('Not defined') for region_pairs in iterlabels: print(region_pairs) try: if compare_code == 0: roi_nat = nf_read_roi_zip(os.path.join( '/mnt/ceph/neuro/labeling/', par[0], consensus_name), c_img.shape) roi_lin = nf_read_roi_zip(os.path.join( '/mnt/ceph/neuro/labeling/', par[0], region_pairs), c_img.shape) else: roi_nat = nf_read_roi_zip(os.path.join( '/mnt/ceph/neuro/labeling/', par[0], region_pairs[0]), c_img.shape) roi_lin = nf_read_roi_zip(os.path.join( '/mnt/ceph/neuro/labeling/', par[0], region_pairs[1]), c_img.shape) print(roi_nat.shape) print(roi_lin.shape) # roi_nat = nf_read_roi_zip(os.path.join('/mnt/ceph/neuro/labeling/',par[0],'regions/natalia_all_regions.zip'),c_img.shape) # roi_lin = nf_read_roi_zip(os.path.join('/mnt/ceph/neuro/labeling/',par[0],'regions/lindsey_all_regions.zip'),c_img.shape) except: print('******************** PROBLEMS WITH ' + par[0])
pl.rc('font', **font) for folder_out in folders_out[:]: projection_img_median = folder_out + '/projections/median_projection.tif' projection_img_correlation = folder_out + '/projections/correlation_image.tif' folder_in = folder_out + '/regions' performance_all = dict() fls = list(glob.glob(folder_in + '/*_nd.zip')) consensus_counter = dict() fl1 = os.path.join(folder_in, 'joined_consensus_active_regions.zip') for fl2 in fls: print([fl1, fl2]) Cn = cm.load(projection_img_correlation) shape = Cn.shape roi_1, names_1 = nf_read_roi_zip(fl1, shape, return_names=True) roi_2, names_2 = nf_read_roi_zip(fl2, shape, return_names=True) # pl.figure() # pl.imshow(np.sum(roi_1,0),cmap = 'gray',vmax=2,alpha=.5) # pl.imshow(np.sum(roi_2,0),cmap = 'hot',vmax=2,alpha=.5) lab1, lab2 = fl1.split('/')[-1][:-4], fl2.split('/')[-1][:-4] # pl.figure(figsize=(15,10)) tp_gt, tp_comp, fn_gt, fp_comp, performance = cm.base.rois.nf_match_neurons_in_binary_masks(roi_1, roi_2, thresh_cost=.7, min_dist=10, print_assignment=False, plot_results=False, Cn=Cn, labels=[lab1, lab2]) performance['tp_gt'] = tp_gt performance['tp_comp'] = tp_comp performance['fn_gt'] = fn_gt performance['fp_comp'] = fp_comp performance_all[fl1, fl2] = performance
ss=None, dview=dview) A_on_thr = A_on_thr > 0 size_neurons = A_on_thr.sum(0) A_on_thr = A_on_thr[:, size_neurons > min_size_neuro] C_on_thr = C_on[size_neurons > min_size_neuro, :116000] print(A_on_thr.shape) C_on_thr = np.array( [CC.reshape([-1, n_frames_per_bin]).max(1) for CC in C_on_thr]) #%% pl.figure() pl.imshow(A_on_thr.sum(-1).reshape(dims_on, order='F')) #%% load labelers roi_rs = nf_read_roi_zip( '/mnt/ceph/neuro/labeling/k53_20160530/regions/sonia_active_regions.zip', Cn.shape) print(roi_rs.shape) roi_bs = nf_read_roi_zip( '/mnt/ceph/neuro/labeling/k53_20160530/regions/lindsey_active_regions.zip', Cn.shape) print(roi_bs.shape) #roi_ds = nf_read_roi_zip('/mnt/ceph/neuro/labeling/J115_2015-12-09_L01/regions/natalia_active_regions_els.zip',Cn.shape) #print(roi_ds.shape) #%% roi_rs = nf_read_roi_zip( '/mnt/ceph/neuro/labeling/J115_2015-12-09_L01/regions/sonia_active_regions_els.zip', Cn.shape) print(roi_rs.shape) roi_bs = nf_read_roi_zip( '/mnt/ceph/neuro/labeling/J115_2015-12-09_L01/regions/lindsey_active_regions_els.zip',
import skimage masks = np.zeros((m1.shape[0], m1.shape[1], mask_new.shape[0])) for i, mm in enumerate(mask_new): rr, cc = skimage.draw.polygon(mm['all_points_y'], mm['all_points_x']) masks[rr, cc, i] = 1 plt.imshow(masks.sum(2)) plt.show() plt.imshow(m1[:, :, 0]) #%% save_folder = '/home/nel/Code/NEL_LAB/Mask_RCNN/datasets/voltage_v1.2_L1_0.5' group = 'train' np.savez(save_folder + '/' + group + '/' + file[:-4] + '_half.npz', img=m1) np.savez(save_folder + '/' + group + '/' + file[:-4] + '_mask.npz', mask=mask_new) #%% transform labels to hdf5 for Viola from caiman.base.rois import nf_read_roi_zip folder = '/home/nel/Code/NEL_LAB/Mask_RCNN/labels/combination_v1.2' save_folder = '/home/nel/Code/NEL_LAB/Mask_RCNN/labels/combination_v1.2_hdf5' files = os.listdir(folder) files = sorted( [file for file in files if 'Fish' not in file and 'IVQ' not in file]) for file in files: path = os.path.join(folder, file) m = nf_read_roi_zip(path, dims=(512, 128)) cm.movie(m).save(os.path.join(save_folder, file[:-4] + '_ROIs.hdf5'))
# Johannes's ROI roi_dir = '/home/nel/Code/Voltage_imaging/exampledata/Johannes/data/' rname = roi_dir+ds+'/ROI_info.npy' y = np.load(rname, allow_pickle=True) dims = X.shape img = np.zeros((dims[0], dims[1])) for i in range(len(y)): img[y[i]['pixel_yx_list'][:,0],y[i]['pixel_yx_list'][:,1]] = 1 plt.figure();plt.imshow(img) # Read my ROI dims = (508, 288)#(360, 256)#(364,320) roi_dir = '/home/nel/Code/VolPy/UNet/ROIs/Johannes' rname = roi_dir+ds+'_RoiSet.zip' from caiman.base.rois import nf_read_roi_zip img = nf_read_roi_zip(rname,dims) plt.figure();plt.imshow(img.sum(axis=0)) # Form training data npz_dir = '/home/nel/Code/VolPy/UNet/npz/Johannes/' X_j = [] dimension = [] for index, file in enumerate(sorted(os.listdir(npz_dir))): if file[-3:] == 'npz': temp = np.load(npz_dir+ file)['arr_0'] dimension.append([temp.shape[0], temp.shape[1]]) temp = Mirror(temp) X_j.append(temp) X_j = np.array(X_j) roi_dir = '/home/nel/Code/VolPy/UNet/ROIs/Johannes/'
for CC in C_on_thr]) #%% pl.figure() pl.imshow(A_on_thr.sum(-1).reshape(dims_on, order='F')) #%% roi_cons = np.load( '/mnt/ceph/neuro/labeling/neurofinder.02.00/regions/joined_consensus_active_regions.npy') print(roi_cons.shape) pl.imshow(roi_cons.sum(0)) #%% roi_cons = np.load( '/mnt/ceph/neuro/labeling/neurofinder.03.00.test/regions/joined_consensus_active_regions.npy') print(roi_cons.shape) pl.imshow(roi_cons.sum(0)) #%% roi_cons = nf_read_roi_zip( '/mnt/ceph/neuro/labeling/neurofinder.02.00/regions/ben_active_regions_nd_natalia_active_regions_nd__sonia_active_regions_nd__lindsey_active_regions_nd_matches.zip', Cn.shape) print(roi_cons.shape) #%% SUE roi_cons = nf_read_roi_zip( '/mnt/ceph/neuro/labeling/k53_20160530/regions/ben_active_regions_nd_natalia_active_regions_nd__sonia_active_regions_nd__lindsey_active_regions_nd_matches.zip', Cn.shape) print(roi_cons.shape) #%% roi_cons = np.load( '/mnt/ceph/neuro/labeling/J115_2015-12-09_L01_ELS/regions/joined_consensus_active_regions.npy') #roi_cons = np.load('/mnt/ceph/neuro/labeling/k53_20160530/regions/joined_consensus_active_regions.npy') print(roi_cons.shape) #%% roi_cons = nf_read_roi_zip( '/mnt/ceph/neuro/labeling/J115_2015-12-09_L01_ELS/regions/ben_active_regions_nd_natalia_active_regions_nd__sonia_active_regions_nd__lindsey_active_regions_nd_matches.zip', Cn.shape) #roi_cons = np.load('/mnt/ceph/neuro/labeling/k53_20160530/regions/joined_consensus_active_regions.npy') print(roi_cons.shape)
#!/usr/bin/env python # -*- coding: utf-8 -*- """ Created on Fri Jul 14 09:42:20 2017 @author: agiovann """ from caiman.base.rois import nf_read_roi_zip # data transformation because of motion correction mismatch in shape #%% sue k37 new_templ = cm.load('projections/median_projection.tif') regions = nf_read_roi_zip( 'regions/joined_consensus_active_regions.zip', new_templ.shape) new_templ = np.pad(new_templ.T, ((7, 8), (4, 3)), mode='constant', constant_values=new_templ.mean()) regions = np.pad(regions.transpose(0, 2, 1), ((0, 0), (7, 8), (4, 3)), mode='constant') np.save('regions/joined_consensus_active_regions', regions) np.save('projections/median_projection', new_templ) #%% if no change corr_img = cm.load('projections/correlation_image.tif') new_templ = cm.load('projections/median_projection.tif') regions = nf_read_roi_zip( 'regions/joined_consensus_active_regions.zip', new_templ.shape) pl.imshow(new_templ / np.nanmax(new_templ), cmap='gray', vmin=0.01, vmax=.3)
frame_name = os.path.join(images_fold, 'image' + str(counter).zfill(5) + '.tif') im = Image.fromarray(fr) im.save(frame_name) z.write(frame_name, os.path.basename(frame_name)) os.remove(frame_name) counter += 1 z.close() dims = np.shape(mov)[1:] regionslist = glob.glob(os.path.join(regions_fold, '*')) for region in regionslist: if 'ben_active_regions_nd.zip' in region: name = os.path.join(regions_dist, 'L4_regions.json') masks = nf_read_roi_zip(region, dims) elif 'lindsey_active_regions_nd.zip' in region: name = os.path.join(regions_dist, 'L3_regions.json') masks = nf_read_roi_zip(region, dims) elif 'sonia_active_regions_nd.zip' in region: name = os.path.join(regions_dist, 'L2_regions.json') masks = nf_read_roi_zip(region, dims) elif 'natalia_active_regions_nd.zip' in region: name = os.path.join(regions_dist, 'L1_regions.json') masks = nf_read_roi_zip(region, dims) elif 'joined_consensus_active_regions.npy' in region: name = os.path.join(regions_dist, 'consensus_regions.json') masks = np.load(region) else: print('discarded ' + region) continue
#%% img_folder = '/home/nel/NEL-LAB Dropbox/NEL/Papers/VolPy/manual_annotations/summary_images/images/' root_folder = '/home/nel/NEL-LAB Dropbox/NEL/Papers/VolPy/manual_annotations/summary_images/output/' #combined_folder = '/home/nel/NEL-LAB Dropbox/NEL/Papers/VolPy/manual_annotations/summary_images/combination' save_img_folder = '/home/nel/NEL-LAB Dropbox/NEL/Papers/VolPy/manual_annotations/summary_images/comparison_v1.2/' annotators = sorted(os.listdir(root_folder)) # select = [0, 1] for select in [[0,1], [0,2], [1,2]]: filenames = sorted(os.listdir(root_folder+annotators[select[0]])) result = {} #filenames1 = sorted(os.listdir(root_folder+annotators[select[1]])) for file in filenames: img = cm.load(os.path.join(img_folder, file[:-4]+'_summary.tif'))[0] dims = img.shape mask0 = nf_read_roi_zip(os.path.join(root_folder, annotators[select[0]], file), dims=dims) * 1. mask1 = nf_read_roi_zip(os.path.join(root_folder, annotators[select[1]], file), dims=dims) * 1. tp_gt, tp_comp, fn_gt, fp_comp, performance_cons_off = nf_match_neurons_in_binary_masks( mask0, mask1, thresh_cost=0.7, min_dist=10, print_assignment=True, plot_results=True, Cn=img, labels=[annotators[select[0]], annotators[select[1]]]) plt.savefig(os.path.join(save_img_folder, annotators[select[0]][0]+'&'+annotators[select[1]][0]+file[:-4]+'.pdf')) plt.close() result[file] = performance_cons_off #%% processed = {} for i in ['f1_score','recall','precision']: #result = eval(i) processed[i] = {} for j in ['L1','TEG','HPC']: