def _segment(self, dataset): def set_z(roi, z): old_mask = roi.mask roi.mask = [ sparse.lil_matrix(old_mask[0].shape, dtype=old_mask[0].dtype) for _ in range(z) ] + [old_mask[0]] rois = ROIList([]) if isinstance(self.strategy, list): if len(self.strategy) != dataset.frame_shape[0]: raise Exception('There is not exactly one strategy per plane.') iterator = list( zip(self.strategy, list(range(dataset.frame_shape[0])))) elif isinstance(self.strategy, SegmentationStrategy): iterator = list( zip(it.repeat(self.strategy), list(range(dataset.frame_shape[0])))) for strategy, plane_idx in iterator: plane_rois = strategy.segment(dataset[:, :, plane_idx]) for roi in plane_rois: set_z(roi, plane_idx) rois.extend(plane_rois) return rois
def _segment(self, dataset): def set_z(roi, z): old_mask = roi.mask return ROI( mask=[sparse.lil_matrix(old_mask[0].shape, old_mask[0].dtype) for _ in range(z-1)] + [old_mask[0]]) rois = ROIList([]) for plane in range(dataset.frame_shape[0]): plane_rois = self.strategy.segment(dataset[:, :, plane]) for roi in plane_rois: set_z(roi, plane) rois.extend(plane_rois) return rois
def _segment(self, dataset): channel = sima.misc.resolve_channels(self._params['channel'], dataset.channel_names) if dataset.savedir is not None: pca_path = os.path.join(dataset.savedir, 'opca_' + str(channel) + '.npz') else: pca_path = None if dataset.savedir is not None: ica_path = os.path.join(dataset.savedir, 'ica_' + str(channel) + '.npz') else: ica_path = None if self._params['verbose']: print('performing PCA...') components = self._params['components'] if isinstance(components, int): components = list(range(components)) _, space_pcs, time_pcs = oPCA.dataset_opca( dataset, channel, components[-1] + 1, path=pca_path) space_pcs = np.real(space_pcs) if self._params['verbose']: print('performing ICA...') st_components = _stica( space_pcs, time_pcs, mu=self._params['mu'], path=ica_path, n_components=space_pcs.shape[-1]) return ROIList([ROI(st_components[..., i]) for i in range(st_components.shape[-1])])
def getRoi(): ds_path = request.form.get('path') label = request.form.get('label') roi_id = request.form.get('id') dataset = ImagingDataset.load(ds_path) convertedRois = {} try: rois = ROIList.load(os.path.join(dataset.savedir, 'rois.pkl'), label=label) except: return jsonify({}) for i, roi in enumerate(rois): if roi.id == roi_id: break roi_points = [] try: for i in xrange(roi.im_shape[0]): roi_points.append([]) except: for i in xrange(np.max(np.array(roi.coords)[:, :, 2])): roi_points.append([]) for poly in roi.polygons: coords = np.array(poly.exterior.coords) if np.all(coords[-1] == coords[0]): coords = coords[:-1] plane = int(coords[0, -1]) coords = coords[:, :2].astype(int).tolist() roi_points[plane].append(coords) return jsonify({roi.id: {'label': roi.label, 'points': roi_points}})
def extract(fpath): fdir = os.path.split(fpath)[0] fname = os.path.splitext(os.path.split(fpath)[1])[0] sima_mc_path = os.path.join(fdir, fname + '_mc.sima') if not os.path.exists(sima_mc_path): raise Exception( 'Data not motion corrected yet; can\'t extract ROI data') rois = ROIList.load( os.path.join(fdir, fname + '_RoiSet.zip'), fmt='ImageJ') # load ROIs as sima polygon objects (list) dataset = sima.ImagingDataset.load(os.path.join( fdir, fname + '_mc.sima')) # reload motion-corrected dataset dataset.add_ROIs(rois, 'from_ImageJ') print('Extracting roi signals from %s' % fdir) signals = dataset.extract(rois) extracted_signals = np.asarray( signals['raw']) # turn signals list into an np array np.save(os.path.join(fdir, fname + '_extractedsignals.npy'), extracted_signals) print('Done with extracting roi signals')
def apply(self, rois, dataset=None): """ Remove overlapping ROIs Parameters ---------- rois : list list of sima.ROI ROIs percent_overlap : float percent of the smaller ROIs total area which must be covered in order for the ROIs to be evaluated as overlapping Returns ------- rois : list A list of sima.ROI ROI objects with the overlapping ROIs combined """ for roi in rois: roi.mask = roi.mask for i in range(len(rois)): # TODO: more efficient strategy for j in [j for j in range(len(rois)) if j != i]: if rois[i] is not None and rois[j] is not None: overlap = np.logical_and(rois[i], rois[j]) small_area = min(np.size(rois[i]), np.size(rois[j])) if len(np.where(overlap)[0]) > \ self.percent_overlap * small_area: new_shape = np.logical_or(rois[i], rois[j]) rois[i] = ROI(mask=new_shape.astype('bool')) rois[j] = None return ROIList(roi for roi in rois if roi is not None)
def apply(self, rois, dataset=None): smoothed, _, _ = SparseROIsFromMasks._find_and_smooth( rois, self.static_threshold, self.smooth_size, self.sign_split, self.n_processes) return ROIList( SparseROIsFromMasks._extract_st_rois(smoothed, self.min_size))
def apply(self, rois, dataset=None): rois_with_ids = [] for index, roi in enumerate(rois): newroi = roi.todict() newroi['id'] = index rois_with_ids.append(newroi) return ROIList(rois_with_ids)
def ROIs(self): try: with open(join(self.savedir, 'rois.pkl'), 'rb') as f: return {label: ROIList(**v) for label, v in pickle.load(f).items()} except (IOError, pickle.UnpicklingError): return {}
def test_extract_roi(self): # make sure number of loaded ROIs (from zip and extracted signals) are correct rois = ROIList.load( os.path.join(self.fdir, self.fname + '_RoiSet.zip'), fmt='ImageJ') # load ROIs as sima polygon objects (list) assert self.sig.shape[1] == 11 # CZ manually drew 11 ROIs assert len(rois) == 11
def apply(self, rois, dataset): channel = sima.misc.resolve_channels(self._channel, dataset.channel_names) processed_im = _processed_image_ca1pc(dataset, channel, self._x_diameter, self._y_diameter)[0] shape = processed_im.shape[:2] ROIs = ROIList([]) for roi in rois: roi_indices = np.nonzero(roi.mask[0]) roi_indices = np.ravel_multi_index(roi_indices, shape) # pixel values in the cut vals = processed_im.flat[roi_indices] # indices of those values below the otsu threshold # if all values are identical, continue without adding an ROI try: roi_indices = roi_indices[vals < threshold_otsu(vals)] except ValueError: continue # apply binary opening and closing to the surviving pixels # expand the shape by 1 in all directions to correct for edge # effects of binary opening/closing twoD_indices = [np.unravel_index(x, shape) for x in roi_indices] mask = np.zeros([x + 2 for x in shape]) for indices in twoD_indices: mask[indices[0] + 1, indices[1] + 1] = 1 mask = ndimage.binary_closing(ndimage.binary_opening(mask)) mask = mask[1:-1, 1:-1] roi_indices = np.where(mask.flat)[0] # label blobs in each cut labeled_array, num_features = measurements.label(mask) for feat in range(num_features): blob_inds = np.where(labeled_array.flat == feat + 1)[0] twoD_indices = [np.unravel_index(x, shape) for x in blob_inds] mask = np.zeros(shape) for x in twoD_indices: mask[x] = 1 ROIs.append(ROI(mask=mask)) return ROIs
def apply(self, rois, dataset): channel = sima.misc.resolve_channels(self._channel, dataset.channel_names) processed_im = _processed_image_ca1pc( dataset, channel, self._x_diameter, self._y_diameter)[0] shape = processed_im.shape[:2] ROIs = ROIList([]) for roi in rois: roi_indices = np.nonzero(roi.mask[0]) roi_indices = np.ravel_multi_index(roi_indices, shape) # pixel values in the cut vals = processed_im.flat[roi_indices] # indices of those values below the otsu threshold # if all values are identical, continue without adding an ROI try: roi_indices = roi_indices[vals < threshold_otsu(vals)] except ValueError: continue # apply binary opening and closing to the surviving pixels # expand the shape by 1 in all directions to correct for edge # effects of binary opening/closing twoD_indices = [np.unravel_index(x, shape) for x in roi_indices] mask = np.zeros([x + 2 for x in shape]) for indices in twoD_indices: mask[indices[0] + 1, indices[1] + 1] = 1 mask = ndimage.binary_closing(ndimage.binary_opening(mask)) mask = mask[1:-1, 1:-1] roi_indices = np.where(mask.flat)[0] # label blobs in each cut labeled_array, num_features = measurements.label(mask) for feat in range(num_features): blob_inds = np.where(labeled_array.flat == feat + 1)[0] twoD_indices = [np.unravel_index(x, shape) for x in blob_inds] mask = np.zeros(shape) for x in twoD_indices: mask[x] = 1 ROIs.append(ROI(mask=mask)) return ROIs
def _segment(self, dataset): def set_z(roi, z): old_mask = roi.mask roi.mask = [sparse.lil_matrix(old_mask[0].shape, dtype=old_mask[0].dtype) for _ in range(z)] + [old_mask[0]] rois = ROIList([]) if isinstance(self.strategy, list): if len(self.strategy) != dataset.frame_shape[0]: raise Exception("There is not exactly one strategy per plane.") iterator = list(zip(self.strategy, list(range(dataset.frame_shape[0])))) elif isinstance(self.strategy, SegmentationStrategy): iterator = list(zip(it.repeat(self.strategy), list(range(dataset.frame_shape[0])))) for strategy, plane_idx in iterator: plane_rois = strategy.segment(dataset[:, :, plane_idx]) for roi in plane_rois: set_z(roi, plane_idx) rois.extend(plane_rois) return rois
def apply(self, rois, dataset=None): SmoothFunc = _SmoothBoundariesParallel(self.tolerance, self.min_verts) if self.n_processes > 1: pool = Pool(processes=self.n_processes) smooth_rois = pool.map(SmoothFunc, rois) pool.close() else: smooth_rois = map(SmoothFunc, rois) return ROIList(smooth_rois)
def _rois_from_cuts(cls, cuts): """Return ROI structures each containing the full extent of a cut. Parameters ---------- cuts : list of sima.normcut.CutRegion The segmented regions identified by normalized cuts. Returns ------- sima.ROI.ROIList ROI structures corresponding to each cut. """ ROIs = ROIList([]) for cut in cuts: if len(cut.indices): mask = np.zeros(cut.shape) for x in cut.indices: mask[np.unravel_index(x, cut.shape)] = 1 ROIs.append(ROI(mask=mask)) return ROIs
def _rois_from_cuts_full(cuts): """Return ROI structures each containing the full extent of a cut. Parameters ---------- cuts : list of sima.normcut.CutRegion The segmented regions identified by normalized cuts. Returns ------- sima.ROI.ROIList ROI structures corresponding to each cut. """ ROIs = ROIList([]) for cut in cuts: if len(cut.indices): mask = np.zeros(cut.shape) for x in cut.indices: mask[np.unravel_index(x, cut.shape)] = 1 ROIs.append(ROI(mask=mask)) return ROIs
def apply(self, rois, dataset=None): if not all(len(r.mask) == 1 for r in rois): raise ValueError('SmoothROIBoundaries applies only to 2D ROIs.') SmoothFunc = _SmoothBoundariesParallel(self.radius) if self.n_processes > 1: pool = Pool(processes=self.n_processes) smooth_rois = pool.map(SmoothFunc, rois) pool.close() else: smooth_rois = map(SmoothFunc, rois) return ROIList([roi[0] for roi in smooth_rois])
def extract_rois((full_path_to_file, individualframename)): # Create SIMA Sequence & ImagingDataset objects from image file(s) or # motion correction if action == 'extract', assume that motion # correction was done on EC2 previously if action == 'both': try: motion_correction((full_path_to_file, individualframename)) except Exception as e: print('Motion correction failed') print e logging.Exception('Motion correction failed') filename = os.path.splitext(os.path.basename(full_path_to_file))[0] dataset = sima.ImagingDataset.load(filename + '_mc.sima') # Obtain ROIs if to_segment: logging.info("Segmenting images for %s..." % filename) # Automated segmentation # Define segmentation method and post processing. segment_approach = sima.segment.PlaneNormalizedCuts() segment_approach.append(sima.segment.SparseROIsFromMasks()) segment_approach.append(sima.segment.SmoothROIBoundaries()) segment_approach.append(sima.segment.MergeOverlapping(threshold=0.5)) # Apply segmentation to dataset rois = dataset.segment(segment_approach) logging.info("Done segmenting images for %s" % filename) print("Done segmenting images for %s" % filename) else: logging.info("Importing ROIs from ImageJ for %s..." % filename) print("Importing ROIs from ImageJ for %s..." % filename) # Load ROIs from ImageJ rois = ROIList.load(filename + '_mc_' + roi_filename, fmt='ImageJ') dataset.add_ROIs(rois, 'from_ImageJ') logging.info("Done importing ROIs for %s" % filename) print("Done importing ROIs for %s" % filename) # Extract signals from ROIs into numpy file signals = dataset.extract(rois) extracted_signals = np.asarray(signals['raw']) np.save(filename + '_extractedsignals', extracted_signals) logging.info("Done extracting signals") print("Done extracting signals")
def deleteRoi(): ds_path = request.form.get('path') label = request.form.get('label') roi_id = request.form.get('roiId') dataset = ImagingDataset.load(ds_path) try: rois = dataset.ROIs[label] except KeyError: return jsonify(result='failed to located ROI List') rois = filter(lambda r: r.id != roi_id, rois) dataset.add_ROIs(ROIList(rois), label=label) return jsonify(result='success')
def extract_dff(directory, plot=False, load_signal=False): hd, tl = os.path.split(directory) if load_signal: dff = np.load(os.path.join(hd, 'DFF')) else: dataset = sima.ImagingDataset.load(directory) rois = ROIList.load(os.path.join(hd, 'manual', 'RoiSet.zip'), fmt='ImageJ') dataset.add_ROIs(rois, 'from_ImageJ') # this immediately saves the ROIs signals = dataset.extract(rois, signal_channel='0', label='0') signals = dataset.signals(channel='0')['0'] # signals are in signals['raw'] # first list index is for each cycle # second index is for each roi within a cycle (third is signal at time t) ## convert each signal into a df/f trace k = 30 baseline_ix = np.array([[0, 90], [240, 320]]) traces = np.stack(signals['raw'], axis=1) dff = np.zeros_like(traces) nroi, ncycles, t = traces.shape # normalize the signal for i, roi in enumerate(traces): for j, cycle in enumerate(roi): cur_mins = [] for b in baseline_ix: ITI = cycle[b[0]:b[1]] sroi = movmean(ITI, 30) cur_mins.append(np.min(sroi)) bsl = np.min(cur_mins) roi_df = (cycle - bsl) / bsl dff[i,j,:] = roi_df # rois x cycles x time # smooth the dff # sroi_df = sp.signal.savgol_filter(roi_df, 29, 3) # sroi_df = sp.ndimage.filters.gaussian_filter1d(roi_df,3) # dff[i,j,:] = sroi_df # rois x cycles x time np.save(os.path.join(hd, 'manual', 'DFF'), dff) # check by plotting if plot: plt.subplots() plt.plot(traces[0,0,:], label='raw signal') plt.plot(dff[0,0,:], label='dF/F') plt.legend() plt.show()
def selectRoi(): ds_path = request.form.get('path') label = request.form.get('label') plane = float(request.form.get('z')) point = Point(float(request.form.get('x')), float(request.form.get('y'))) dataset = ImagingDataset.load(ds_path) rois = ROIList.load(os.path.join(dataset.savedir, 'rois.pkl'), label=label) for roi in rois: for poly in roi.polygons: z_coord = np.array(poly.exterior.coords)[0, 2] if z_coord == plane or plane == -1: if poly.contains(point): return jsonify(label=roi.label, id=roi.id) return jsonify({'error': 'roi not found'})
def _segment(self, dataset): channel = sima.misc.resolve_channels(self._params['channel'], dataset.channel_names) if dataset.savedir is not None: pca_path = os.path.join(dataset.savedir, 'opca_' + str(channel) + '.npz') else: pca_path = None if dataset.savedir is not None: ica_path = os.path.join(dataset.savedir, 'ica_' + str(channel) + '.npz') else: ica_path = None if self._params['verbose']: print('performing PCA...') components = self._params['components'] if isinstance(components, int): components = list(range(components)) _, space_pcs, time_pcs = oPCA.dataset_opca( dataset, channel, components[-1] + 1, path=pca_path) space_pcs = np.real(space_pcs) # Remove components greater than the number of PCs returned # in case more components were asked for than the number of # independent dimensions in the dataset. components = [c for c in components if c < time_pcs.shape[1]] if self._params['verbose']: print('performing ICA...') st_components = _stica( space_pcs, time_pcs, mu=self._params['mu'], path=ica_path, n_components=space_pcs.shape[-1]) return ROIList([ROI(st_components[..., i]) for i in range(st_components.shape[-1])])
def getRois(): ds_path = request.form.get('path') label = request.form.get('label') dataset = ImagingDataset.load(ds_path) convertedRois = {} rois = ROIList.load(os.path.join(dataset.savedir, 'rois.pkl'), label=label) for i, roi in enumerate(rois): if roi.label is None: roi.label = i convertedRois[roi.label] = {} for poly in roi.polygons: coords = np.array(poly.exterior.coords) plane = int(coords[0, -1]) #coords = list(coords[:,:2].ravel()) coords = coords[:, :2].tolist() try: convertedRois[roi.label][plane].append(coords) except KeyError: convertedRois[roi.label][plane] = [coords] return jsonify(**convertedRois)
def getRois(): ds_path = request.form.get("path") label = request.form.get("label") dataset = ImagingDataset.load(ds_path) convertedRois = {} rois = ROIList.load(os.path.join(dataset.savedir, "rois.pkl"), label=label) for i, roi in enumerate(rois): if roi.label is None: roi.label = i convertedRois[roi.label] = {} for poly in roi.polygons: coords = np.array(poly.exterior.coords) plane = int(coords[0, -1]) # coords = list(coords[:,:2].ravel()) coords = coords[:, :2].tolist() try: convertedRois[roi.label][plane].append(coords) except KeyError: convertedRois[roi.label][plane] = [coords] return jsonify(**convertedRois)
def _remove_overlapping(rois, percent_overlap=0.9): """ Remove overlapping ROIs Parameters ---------- rois : list list of sima.ROI ROIs percent_overlap : float percent of the smaller ROIs total area which must be covered in order for the ROIs to be evaluated as overlapping Returns ------- rois : list A list of sima.ROI ROI objects with the overlapping ROIs combined """ if percent_overlap > 0 and percent_overlap <= 1: for roi in rois: roi.mask = roi.mask for i in xrange(len(rois)): for j in [j for j in xrange(len(rois)) if j != i]: if rois[i] is not None and rois[j] is not None: overlap = np.logical_and(rois[i].mask.toarray(), rois[j].mask.toarray()) small_area = np.min((rois[i].mask.size, rois[j].mask.size)) if len(np.where(overlap)[0]) > \ percent_overlap * small_area: new_shape = np.logical_or(rois[i].mask.toarray(), rois[j].mask.toarray()) rois[i] = ROI(mask=new_shape.astype('bool'), im_shape=rois[i].mask.shape) rois[j] = None return ROIList(roi for roi in rois if roi is not None)
def updateRoi(): ds_path = request.form.get('path') label = request.form.get('label') points = json.loads(request.form.get('points')) roi_label = request.form.get('roiLabel') roi_id = request.form.get('roiId') dataset = ImagingDataset.load(ds_path) roi_data = [] for i, plane in enumerate(points): if plane is None or not len(plane): continue array_dat = np.array(plane) z_dims = i * np.ones((array_dat.shape[:2] + (1, ))) plane_data = np.concatenate((array_dat, z_dims), axis=2) roi_data.extend(list(plane_data)) if len(roi_data) == 0: return jsonify(result="no polygons to save") for poly in roi_data: if poly.shape[0] < 3: raise Exception("unable to store polygon with less then 3 points") roi = ROI(polygons=roi_data, im_shape=dataset.frame_shape[:3]) roi.label = roi_label roi.id = roi_id try: rois = dataset.ROIs[label] except KeyError: rois = [] rois = filter(lambda r: r.id != roi_id, rois) rois.append(roi) dataset.add_ROIs(ROIList(rois), label=label) return jsonify(result='success')
def setRoiLabel(): ds_path = request.form.get('path') #old_label = request.form.get('oldLabel') old_label = '' new_label = request.form.get('newLabel') if new_label == '': new_label = 'rois' dataset = ImagingDataset.load(ds_path) if (old_label != ''): rois = dataset.ROIs[old_label] else: rois = ROIList([]) dataset.add_ROIs(rois, label=new_label) labels = dataset.ROIs.keys() labels.extend( map(os.path.basename, glob.glob(os.path.join(ds_path, 'ica*.npz')))) labels.extend( map(os.path.basename, glob.glob(os.path.join(ds_path, 'opca*.npz')))) return jsonify({'labels': labels})
def ROILoadFromDir(_path, _dir): fullRAP_roi_list = [] for i in range(0, len(_dir)): fullRAP_roi_list.append(ROIList.load(_path + _dir[i], fmt='ImageJ')) return fullRAP_roi_list
def process_data(haussio_data, mask=None, p=2, nrois_init=200): fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat' tiffs_to_cnmf(haussio_data, mask) sys.stdout.write('Loading from {0}... '.format( haussio_data.dirname_comp + '_Y*.npy')) Y = np.load(haussio_data.dirname_comp + '_Y.npy', mmap_mode='r') d1, d2, T = Y.shape if not os.path.exists(fn_cnmf): cse.utilities.stop_server() sys.stdout.flush() t0 = time.time() Yr = np.load(haussio_data.dirname_comp + '_Yr.npy', mmap_mode='r') sys.stdout.write('took {0:.2f} s\n'.format(time.time()-t0)) # how to subdivide the work among processes n_pixels_per_process = d1*d2/NCPUS options = cse.utilities.CNMFSetParms(Y, K=nrois_init, p=p, gSig=[9, 9]) options['preprocess_params']['n_processes'] = NCPUS options['preprocess_params'][ 'n_pixels_per_process'] = n_pixels_per_process options['init_params']['nIter'] = 10 options['init_params']['maxIter'] = 10 options['init_params']['use_hals'] = False options['spatial_params']['n_processes'] = NCPUS options['spatial_params'][ 'n_pixels_per_process'] = n_pixels_per_process options['temporal_params']['n_processes'] = NCPUS options['temporal_params'][ 'n_pixels_per_process'] = n_pixels_per_process cse.utilities.start_server(NCPUS) t0 = time.time() sys.stdout.write("Preprocessing... ") sys.stdout.flush() Yr, sn, g = cse.preprocess_data(Yr, **options['preprocess_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 224.94s t0 = time.time() sys.stdout.write("Initializing components... ") sys.stdout.flush() Ain, Cin, b_in, f_in, center = cse.initialize_components( Y, **options['init_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 2281.37s t0 = time.time() sys.stdout.write("Updating spatial components... ") sys.stdout.flush() A, b, Cin = cse.update_spatial_components( Yr, Cin, f_in, Ain, sn=sn, **options['spatial_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 252.57s t0 = time.time() sys.stdout.write("Updating temporal components... ") sys.stdout.flush() C, f, S, bl, c1, neurons_sn, g, YrA = \ cse.update_temporal_components( Yr, A, b, Cin, f_in, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 455.14s t0 = time.time() sys.stdout.write("Merging ROIs... ") sys.stdout.flush() A_m, C_m, nr_m, merged_ROIs, S_m, bl_m, c1_m, sn_m, g_m = \ cse.merge_components( Yr, A, b, C, f, S, sn, options['temporal_params'], options['spatial_params'], bl=bl, c1=c1, sn=neurons_sn, g=g, thr=0.7, mx=100, fast_merge=True) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 702.55s t0 = time.time() sys.stdout.write("Updating spatial components... ") sys.stdout.flush() A2, b2, C2 = cse.update_spatial_components( Yr, C_m, f, A_m, sn=sn, **options['spatial_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 77.16s t0 = time.time() sys.stdout.write("Updating temporal components... ") sys.stdout.flush() C2, f2, S2, bl2, c12, neurons_sn2, g21, YrA = \ cse.update_temporal_components( Yr, A2, b2, C2, f, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 483.41s # A: spatial components (ROIs) # C: denoised [Ca2+] # YrA: residuals ("noise") # S: Spikes savemat(fn_cnmf, {"A": A2, "C": C2, "YrA": YrA, "S": S2, "bl": bl2}) else: resdict = loadmat(fn_cnmf) A2 = resdict["A"] C2 = resdict["C"] YrA = resdict["YrA"] S2 = resdict["S"] bl2 = resdict["bl"] proj_fn = haussio_data.dirname_comp + "_proj.npy" if not os.path.exists(proj_fn): zproj = utils.zproject(np.transpose(Y, (2, 0, 1))) np.save(proj_fn, zproj) else: zproj = np.load(proj_fn) # DF_F, DF = cse.extract_DF_F(Y.reshape(d1*d2, T), A2, C2) t0 = time.time() sys.stdout.write("Ordering components... ") sys.stdout.flush() A_or, C_or, srt = cse.order_components(A2, C2) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) cse.utilities.stop_server() polygons = contour(A2, d1, d2, thr=0.9) rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons]) return rois, C2, haussio_data, zproj, S2, Y, YrA
def _load_version0(path): """Load a SIMA 0.x dataset Parameters ---------- path : str The path to the original saved dataset, ending in .sima Examples -------- >>> from sima.misc import example_data >>> from sima.misc.convert import _load_version0 >>> ds = _load_version0(example_data()) """ def parse_channel(channel): """Parse an old format channel stored a dictionary Parameters ---------- channel : dict Returns ------- result : sima.Sequence A sequence equivalent to the old format channel. """ _resolve_paths(channel, path) klass = channel.pop('__class__') if klass == 'sima.iterables.MultiPageTIFF': result = Sequence.create('TIFF', channel['path']) try: clip = channel['clip'] except KeyError: pass else: if clip is not None: s = (slice(None), slice(None)) + tuple( slice(*[None if x is 0 else x for x in dim]) for dim in clip) result = result[s] return result elif klass == 'sima.iterables.HDF5': raise Exception('TODO') else: raise Exception('Format not recognized.') def parse_sequence(sequence): channels = [parse_channel(c) for c in sequence] return Sequence.join(channels) with open(os.path.join(path, 'dataset.pkl'), 'rb') as f: unpickler = Unpickler(f) dataset_dict = unpickler.load() iterables = dataset_dict.pop('iterables') sequences = [parse_sequence(seq) for seq in iterables] # Apply displacements if they exist try: with open(os.path.join(path, 'displacements.pkl'), 'rb') as f: displacements = pkl.load(f) except IOError: pass else: assert all(np.all(d >= 0) for d in displacements) max_disp = np.max(list(chain(*displacements)), axis=0) frame_shape = np.array(sequences[0].shape)[1:] frame_shape[1:3] += max_disp sequences = [ s.apply_displacements(d.reshape(s.shape[:3] + (2,)), frame_shape) for s, d in zip(sequences, displacements)] try: trim_coords = dataset_dict.pop('_lazy__trim_coords') except KeyError: try: trim_criterion = dataset_dict.pop('trim_criterion') except KeyError: pass else: raise Exception( 'Parsing of trim_criterion ' + str(trim_criterion) + ' not yet implemented') else: sequences = [s[:, :, trim_coords[0][0]:trim_coords[1][0], trim_coords[0][1]:trim_coords[1][1]] for s in sequences] ds = ImagingDataset(sequences, None) ds.savedir = path # Add ROIs if they exist try: with open(os.path.join(path, 'rois.pkl'), 'rb') as f: rois = pkl.load(f) except IOError: pass else: roi_lists = {} for label, roi_list_dict in rois.iteritems(): roi_list = [] for roi in roi_list_dict['rois']: mask = roi['mask'] polygons = roi['polygons'] if mask is not None: new_roi = ROI(mask=mask) else: new_roi = ROI(polygons=polygons) new_roi.id = roi['id'] new_roi.label = roi['label'] new_roi.tags = roi['tags'] new_roi.im_shape = roi['im_shape'] roi_list.append(new_roi) roi_lists[label] = ROIList(roi_list) roi_lists[label].timestamp = roi_list_dict['timestamp'] for label, roi_list in roi_lists.iteritems(): ds.add_ROIs(roi_list, label=label) return ds
def apply(self, rois, dataset=None): return ROIList([r for r in rois if self._valid(r)])
def _segment(self, dataset): channel = sima.misc.resolve_channels(self._params.channel, dataset.channel_names) if dataset.savedir is not None: pca_path = os.path.join(dataset.savedir, 'opca_' + str(channel) + '.npz') else: pca_path = None if dataset.savedir is not None: ica_path = os.path.join(dataset.savedir, 'ica_' + str(channel) + '.npz') else: ica_path = None if self._params.verbose: print 'performing PCA...' if isinstance(self._params.components, int): self._params.components = range(self._params.components) _, space_pcs, time_pcs = _OPCA(dataset, channel, self._params.components[-1] + 1, path=pca_path) space_pcs = np.real( space_pcs.reshape(dataset.frame_shape[1:3] + (space_pcs.shape[2], ))) space_pcs = np.array( [space_pcs[:, :, i] for i in self._params.components]).transpose( (1, 2, 0)) time_pcs = np.array([time_pcs[:, i] for i in self._params.components]).transpose( (1, 0)) if self._params.verbose: print 'performing ICA...' st_components = _stica(space_pcs, time_pcs, mu=self._params.mu, path=ica_path, n_components=space_pcs.shape[2]) if self._params.x_smoothing > 0 or self._params.static_threshold > 0: accepted, _, _ = _find_useful_components( st_components, self._params.static_threshold, x_smoothing=self._params.x_smoothing) if self._params.min_area > 0 or self._params.spatial_sep: rois = _extract_st_rois(accepted, min_area=self._params.min_area, spatial_sep=self._params.spatial_sep) if self._params.smooth_rois: if self._params.verbose: print 'smoothing ROIs...' rois = [_smooth_roi(roi)[0] for roi in rois] if self._params.verbose: print 'removing overlapping ROIs...' rois = _remove_overlapping( rois, percent_overlap=self._params.overlap_per) else: rois = [ ROI(st_components[:, :, i]) for i in xrange(st_components.shape[2]) ] return ROIList(rois)
def process_data_patches(haussio_data, mask=None, p=2, nrois_init=400, roi_iceberg=0.9): fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat' tiffs_to_cnmf(haussio_data, mask) tmpdirname_comp = os.path.join(tempfile.gettempdir(), haussio_data.dirname_comp) try: os.makedirs(os.path.dirname(tmpdirname_comp)) except OSError: pass sys.stdout.write('Loading from {0}... '.format(tmpdirname_comp + '_Y*.npy')) Y = np.load(tmpdirname_comp + '_Y.npy', mmap_mode='r') d1, d2, T = Y.shape if not os.path.exists(fn_cnmf): cse.utilities.stop_server() sys.stdout.flush() t0 = time.time() fname_new = get_mmap_name(tmpdirname_comp, d1, d2, T) Yr, _, _ = cse.utilities.load_memmap(fname_new) sys.stdout.write('took {0:.2f} s\n'.format(time.time() - t0)) # how to subdivide the work among processes n_pixels_per_process = d1 * d2 / NCPUS_PATCHES sys.stdout.flush() cse.utilities.stop_server() cse.utilities.start_server() cl = Client() dview = cl[:NCPUS_PATCHES] rf = int( np.ceil(np.sqrt(d1 * d2 / 4 / NCPUS_PATCHES)) ) # half-size of the patches in pixels. rf=25, patches are 50x50 sys.stdout.write("Patch size: {0} * {0} = {1}\n".format( rf * 2, rf * rf * 4)) stride = int(rf / 5) # amounpl of overlap between the patches in pixels t0 = time.time() sys.stdout.write("CNMF patches... ") sys.stdout.flush() options_patch = cse.utilities.CNMFSetParms(Y, NCPUS_PATCHES, p=0, gSig=[16, 16], K=nrois_init / NCPUS_PATCHES, ssub=1, tsub=8, thr=0.8) A_tot, C_tot, YrA_tot, b, f, sn_tot, opt_out = cse.map_reduce.run_CNMF_patches( fname_new, (d1, d2, T), options_patch, rf=rf, stride=stride, dview=dview, memory_fact=4.0) sys.stdout.write(' took {0:.2f} s\n'.format(time.time() - t0)) options = cse.utilities.CNMFSetParms(Y, NCPUS_PATCHES, K=A_tot.shape[-1], p=p, gSig=[16, 16], ssub=1, tsub=1) pix_proc = np.minimum( np.int((d1 * d2) / NCPUS_PATCHES / (T / 2000.)), np.int((d1 * d2) / NCPUS_PATCHES)) # regulates the amount of memory used options['spatial_params']['n_pixels_per_process'] = pix_proc options['temporal_params']['n_pixels_per_process'] = pix_proc t0 = time.time() sys.stdout.write("Merging ROIs... ") sys.stdout.flush() A_m, C_m, nr_m, merged_ROIs, S_m, bl_m, c1_m, sn_m, g_m = \ cse.merge_components( Yr, A_tot, [], np.array(C_tot), [], np.array(C_tot), [], options['temporal_params'], options['spatial_params'], dview=dview, thr=options['merging']['thr'], mx=np.Inf) sys.stdout.write(' took {0:.2f} s\n'.format(time.time() - t0)) options['temporal_params']['p'] = 0 options['temporal_params'][ 'fudge_factor'] = 0.96 #change ifdenoised traces time constant is wrong options['temporal_params']['backend'] = 'ipyparallel' t0 = time.time() sys.stdout.write("Updating temporal components... ") sys.stdout.flush() C_m, f_m, S_m, bl_m, c1_m, neurons_sn_m, g2_m, YrA_m = \ cse.temporal.update_temporal_components( Yr, A_m, np.atleast_2d(b).T, C_m, f, dview=dview, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time() - t0)) # 483.41s # 2016-05-24: 74.81s t0 = time.time() sys.stdout.write("Evaluating components... ") sys.stdout.flush() traces = C_m + YrA_m Npeaks = 10 final_frate = 1.0 / haussio_data.dt tB = np.minimum(-2, np.floor(-5. / 30 * final_frate)) tA = np.maximum(5, np.ceil(25. / 30 * final_frate)) fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, sign_sam =\ cse.utilities.evaluate_components( Y, traces, A_m, C_m, b, f_m, remove_baseline=True, N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks, tB=tB, tA=tA, thresh_C=0.3) idx_components_r = np.where(r_values >= .5)[0] idx_components_raw = np.where(fitness_raw < -20)[0] idx_components_delta = np.where(fitness_delta < -10)[0] idx_components = np.union1d(idx_components_r, idx_components_raw) idx_components = np.union1d(idx_components, idx_components_delta) A_m = A_m[:, idx_components] C_m = C_m[idx_components, :] sys.stdout.write(' took {0:.2f} s\n'.format(time.time() - t0)) t0 = time.time() sys.stdout.write("Updating spatial components... ") sys.stdout.flush() A2, b2, C2 = cse.spatial.update_spatial_components( Yr, C_m, f, A_m, sn=sn_tot, dview=dview, **options['spatial_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time() - t0)) # 77.16s # 2016-05-24: 99.22s options['temporal_params']['p'] = p options['temporal_params'][ 'fudge_factor'] = 0.96 #change ifdenoised traces time constant is wrong C2, f2, S2, bl2, c12, neurons_sn2, g21, YrA = \ cse.temporal.update_temporal_components( Yr, A2, b2, C2, f, dview=dview, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) traces = C2 + YrA fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, sign_sam =\ cse.utilities.evaluate_components( Y, traces, A2, C2, b2, f2, remove_baseline=True, N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks, tB=tB, tA=tA, thresh_C=0.3) idx_components_r = np.where(r_values >= .6)[0] idx_components_raw = np.where(fitness_raw < -60)[0] idx_components_delta = np.where(fitness_delta < -20)[0] idx_components = np.union1d(idx_components_r, idx_components_raw) idx_components = np.union1d(idx_components, idx_components_delta) A2 = A2.tocsc()[:, idx_components] C2 = C2[idx_components, :] YrA = YrA[idx_components, :] S2 = S2[idx_components, :] # A: spatial components (ROIs) # C: denoised [Ca2+] # YrA: residuals ("noise") # S: Spikes savemat(fn_cnmf, {"A": A2, "C": C2, "YrA": YrA, "S": S2, "bl": bl2}) else: resdict = loadmat(fn_cnmf) A2 = resdict["A"] C2 = resdict["C"] YrA = resdict["YrA"] S2 = resdict["S"] bl2 = resdict["bl"] proj_fn = haussio_data.dirname_comp + "_proj.npy" if not os.path.exists(proj_fn): zproj = utils.zproject(np.transpose(Y, (2, 0, 1))) np.save(proj_fn, zproj) else: zproj = np.load(proj_fn) # DF_F, DF = cse.extract_DF_F(Y.reshape(d1*d2, T), A2, C2) # t0 = time.time() # sys.stdout.write("Ordering components... ") # sys.stdout.flush() # A_or, C_or, srt = cse.order_components(A2, C2) # sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) cse.utilities.stop_server() polygons = contour(A2, d1, d2, thr=roi_iceberg) rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons]) return rois, C2, zproj, S2, Y, YrA
def process_data(haussio_data, mask=None, p=2, nrois_init=400, roi_iceberg=0.9, merge_unconnected=None): if mask is not None: raise RuntimeError("mask not supported in cnmf.process_data") fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat' shapefn = os.path.join(haussio_data.dirname_comp, haussio.THOR_RAW_FN[:-3] + "shape.npy") shape = np.load(shapefn) if len(shape) == 5: d1, d2 = shape[2], shape[3] else: d1, d2 = shape[1], shape[2] fn_mmap = get_mmap_name(haussio_data.dirname_comp + os.path.sep + 'Yr', d1, d2, shape[0]) tiffs_to_cnmf(haussio_data) if os.path.exists(fn_cnmf): resdict = loadmat(fn_cnmf) if "dFoF" in resdict.keys(): A2 = resdict["A"] C2 = resdict["C"] YrA = resdict["YrA"] S2 = resdict["S"] dFoF = resdict["dFoF"] bl2 = resdict["bl"] f = resdict["f"] images = haussio_data.read_raw().squeeze() else: dFoF = None if not os.path.exists(fn_cnmf) or dFoF is None: c, dview, n_processes = cm.cluster.setup_cluster( backend='multiprocessing', n_processes=NCPUS, single_thread=False) Yr, dims, T = cm.load_memmap(fn_mmap, 'r+') d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') fr = 1.0 / haussio_data.dt # imaging rate in frames per second\n", decay_time = 0.4 # length of a typical transient in seconds\n", # parameters for source extraction and deconvolution\n", bord_px_els = 32 # maximum shift to be used for trimming against NaNs p = 1 # order of the autoregressive system\n", gnb = 2 # number of global background components\n", merge_thresh = 0.8 # merging threshold, max correlation allowed\n", rf = int( np.round(np.sqrt(d1 * d2) / nrois_init) ) # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50\n", if rf < 16: rf = 16 stride_cnmf = 6 # amount of overlap between the patches in pixels\n", npatches = np.round(d1 / (rf * 2) * d2 / (rf * 2)) K = nrois_init / npatches # number of components per patch\n", if K < 2: K = 2 print(rf, npatches, K) gSig = [8, 8] # expected half size of neurons\n", init_method = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf')\n", is_dendrites = False # flag for analyzing dendritic data\n", alpha_snmf = None # sparsity penalty for dendritic data analysis through sparse NMF\n", # parameters for component evaluation\n", min_SNR = 2.5 # signal to noise ratio for accepting a component\n", rval_thr = 0.8 # space correlation threshold for accepting a component\n", cnn_thr = 0.8 # threshold for CNN based classifier" cnm = caiman_cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview = dview, min_SNR=min_SNR, r_values_min = rval_thr, use_cnn = False, thresh_cnn_lowest = cnn_thr) A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[ idx_components], cnm.b, cnm.f cnm2 = caiman_cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) if merge_unconnected is not None: idx_merge = [] for nroi, ca_roi in enumerate(cnm2.C): for nroi_compare_counter, ca_roi_compare in enumerate( cnm2.C[nroi + 1:]): nroi_compare = nroi_compare_counter + nroi + 1 if nroi_compare not in idx_merge: correls = np.correlate(ca_roi, ca_roi_compare, mode='same') correls /= np.sqrt( np.dot(ca_roi, ca_roi) * np.dot(ca_roi_compare, ca_roi_compare)) if correls.max() > merge_unconnected: print("Merging ", nroi_compare) idx_merge.append(nroi_compare) idx_no_merge = [ idx for idx in range(cnm2.C.shape[0]) if idx not in idx_merge ] else: idx_no_merge = range(cnm2.C.shape[0]) A2 = cnm2.A[:, idx_no_merge].tocsc() C2 = cnm2.C[idx_no_merge] YrA = cnm2.YrA[idx_no_merge] S2 = cnm2.S[idx_no_merge] dFoF = cnm2.detrend_df_f(frames_window=300)[idx_no_merge] # A: spatial components (ROIs) # C: denoised [Ca2+] # YrA: residuals ("noise", i.e. traces = C+YrA) # S: Spikes # f: temporal background savemat( fn_cnmf, { "A": A2, "C": C2, "YrA": YrA, "S": S2, "dFoF": dFoF, "bl": cnm2.b, "f": cnm2.f }) dview.terminate() cm.stop_server() proj_fn = haussio_data.dirname_comp + "_proj.npy" if not os.path.exists(proj_fn): zproj = utils.zproject(images) np.save(proj_fn, zproj) else: zproj = np.load(proj_fn) logfiles = glob.glob("*LOG*") for logfile in logfiles: try: os.unlink(logfile) except OSError: pass polygons = contour(A2, images.shape[1], images.shape[2], thr=roi_iceberg) rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons]) return rois, C2, zproj, S2, images, YrA
def _rois_from_cuts_ca1pc(cuts, im_set, circularity_threhold=0.5, min_roi_size=20, min_cut_size=30, channel=0, x_diameter=8, y_diameter=8): """Return ROI structures containing CA1 pyramidal cell somata. Parameters ---------- cuts : list of sima.normcut.CutRegion The segmented regions identified by normalized cuts. circularity_threhold : float ROIs with circularity below threshold are discarded. Default: 0.5. min_roi_size : int, optional ROIs with fewer than min_roi_size pixels are discarded. Default: 20. min_cut_size : int, optional No ROIs are made from cuts with fewer than min_cut_size pixels. Default: 30. channel : int, optional The index of the channel to be used. x_diameter : int, optional The estimated x-diameter of the nuclei in pixels y_diameter : int, optional The estimated y_diameter of the nuclei in pixels Returns ------- sima.ROI.ROIList ROI structures each corresponding to a CA1 pyramidal cell soma. """ processed_im = _processed_image_ca1pc(im_set, channel, x_diameter, y_diameter) shape = processed_im.shape[:2] ROIs = ROIList([]) for cut in cuts: if len(cut.indices) > min_cut_size: # pixel values in the cut vals = processed_im.flat[cut.indices] # indices of those values below the otsu threshold # if all values are identical, continue without adding an ROI try: roi_indices = cut.indices[vals < threshold_otsu(vals)] except ValueError: continue # apply binary opening and closing to the surviving pixels # expand the shape by 1 in all directions to correct for edge # effects of binary opening/closing twoD_indices = [np.unravel_index(x, shape) for x in roi_indices] mask = np.zeros([x + 2 for x in shape]) for indices in twoD_indices: mask[indices[0] + 1, indices[1] + 1] = 1 mask = ndimage.binary_closing(ndimage.binary_opening(mask)) mask = mask[1:-1, 1:-1] roi_indices = np.where(mask.flat)[0] # label blobs in each cut labeled_array, num_features = label(mask) for feat in range(num_features): blob_inds = np.where(labeled_array.flat == feat + 1)[0] # Apply min ROI size threshold if len(blob_inds) > min_roi_size: twoD_indices = [np.unravel_index(x, shape) for x in blob_inds] mask = np.zeros(shape) for x in twoD_indices: mask[x] = 1 # APPLY CIRCULARITY THRESHOLD poly_pts = np.array(mask2poly(mask)[0].exterior.coords) p = 0 for x in range(len(poly_pts) - 1): p += np.linalg.norm(poly_pts[x] - poly_pts[x + 1]) shape_area = len(roi_indices) circle_area = np.square(p) / (4 * np.pi) if shape_area / circle_area > circularity_threhold: ROIs.append(ROI(mask=mask)) return ROIs
def process_data(haussio_data, mask=None, p=2, nrois_init=400, roi_iceberg=0.9): if mask is not None: raise RuntimeError("mask not supported in cnmf.process_data") fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat' shapefn = os.path.join(haussio_data.dirname_comp, haussio.THOR_RAW_FN[:-3] + "shape.npy") shape = np.load(shapefn) if len(shape) == 5: d1, d2 = shape[2], shape[3] fn_mmap = get_mmap_name('Yr', shape[2], shape[3], shape[0]) else: d1, d2 = shape[1], shape[2] fn_mmap = get_mmap_name('Yr', shape[1], shape[2], shape[0]) fn_mmap = os.path.join(haussio_data.dirname_comp, fn_mmap) print(fn_mmap, os.path.exists(fn_mmap), d1, d2) if not os.path.exists(fn_cnmf): # fn_raw = os.path.join(haussio_data.dirname_comp, haussio.THOR_RAW_FN) fn_sima = haussio_data.dirname_comp + '.sima' fnames = [ fn_sima, ] fnames.sort() print(fnames) fnames = fnames final_frate = 1.0 / haussio_data.dt downsample_factor = 1 # use .2 or .1 if file is large and you want a quick answer final_frate *= downsample_factor c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) idx_xy = None base_name = 'Yr' name_new = cm.save_memmap_each(fnames, dview=dview, base_name=base_name, resize_fact=(1, 1, downsample_factor), remove_init=0, idx_xy=idx_xy) name_new.sort() print(name_new) if len(name_new) > 1: fname_new = cm.save_memmap_join(name_new, base_name='Yr', n_chunks=12, dview=dview) else: sys.stdout.write('One file only, not saving\n') fname_new = name_new[0] print("fname_new: " + fname_new) Yr, dims, T = cm.load_memmap(fname_new) Y = np.reshape(Yr, dims + (T, ), order='F') Cn = cm.local_correlations(Y) K = nrois_init # number of neurons expected per patch gSig = [15, 15] # expected half size of neurons merge_thresh = 0.8 # merging threshold, max correlation allowed p = 2 #order of the autoregressive system options = caiman_cnmf.utilities.CNMFSetParms(Y, NCPUS, p=p, gSig=gSig, K=K, ssub=2, tsub=2) Yr, sn, g, psx = caiman_cnmf.pre_processing.preprocess_data( Yr, dview=dview, **options['preprocess_params']) Atmp, Ctmp, b_in, f_in, center = caiman_cnmf.initialization.initialize_components( Y, **options['init_params']) Ain, Cin = Atmp, Ctmp A, b, Cin, f_in = caiman_cnmf.spatial.update_spatial_components( Yr, Cin, f_in, Ain, sn=sn, dview=dview, **options['spatial_params']) options['temporal_params'][ 'p'] = 0 # set this to zero for fast updating without deconvolution C, A, b, f, S, bl, c1, neurons_sn, g, YrA = caiman_cnmf.temporal.update_temporal_components( Yr, A, b, Cin, f_in, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) A_m, C_m, nr_m, merged_ROIs, S_m, bl_m, c1_m, sn_m, g_m = caiman_cnmf.merging.merge_components( Yr, A, b, C, f, S, sn, options['temporal_params'], options['spatial_params'], dview=dview, bl=bl, c1=c1, sn=neurons_sn, g=g, thr=merge_thresh, mx=50, fast_merge=True) A2, b2, C2, f = caiman_cnmf.spatial.update_spatial_components( Yr, C_m, f, A_m, sn=sn, dview=dview, **options['spatial_params']) options['temporal_params'][ 'p'] = p # set it back to original value to perform full deconvolution C2, A2, b2, f2, S2, bl2, c12, neurons_sn2, g21, YrA = caiman_cnmf.temporal.update_temporal_components( Yr, A2, b2, C2, f, dview=dview, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) tB = np.minimum(-2, np.floor(-5. / 30 * final_frate)) tA = np.maximum(5, np.ceil(25. / 30 * final_frate)) Npeaks = 10 traces = C2 + YrA fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = \ evaluate_components( Y, traces, A2, C2, b2, f2, final_frate, remove_baseline=True, N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks, thresh_C=0.3) idx_components_r = np.where(r_values >= .6)[0] idx_components_raw = np.where(fitness_raw < -60)[0] idx_components_delta = np.where(fitness_delta < -20)[0] min_radius = gSig[0] - 2 masks_ws, idx_blobs, idx_non_blobs = extract_binary_masks_blob( A2.tocsc(), min_radius, dims, num_std_threshold=1, minCircularity=0.6, minInertiaRatio=0.2, minConvexity=.8) idx_components = np.union1d(idx_components_r, idx_components_raw) idx_components = np.union1d(idx_components, idx_components_delta) idx_blobs = np.intersect1d(idx_components, idx_blobs) idx_components_bad = np.setdiff1d(range(len(traces)), idx_components) A2 = A2.tocsc()[:, idx_components] C2 = C2[idx_components, :] YrA = YrA[idx_components, :] S2 = S2[idx_components, :] # A: spatial components (ROIs) # C: denoised [Ca2+] # YrA: residuals ("noise", i.e. traces = C+YrA) # S: Spikes savemat(fn_cnmf, {"A": A2, "C": C2, "YrA": YrA, "S": S2, "bl": bl2}) else: resdict = loadmat(fn_cnmf) A2 = resdict["A"] C2 = resdict["C"] YrA = resdict["YrA"] S2 = resdict["S"] bl2 = resdict["bl"] Yr, dims, T = cm.load_memmap(fn_mmap) dims = dims[1:] Y = np.reshape(Yr, dims + (T, ), order='F') proj_fn = haussio_data.dirname_comp + "_proj.npy" if not os.path.exists(proj_fn): zproj = utils.zproject(np.transpose(Y, (2, 0, 1))) np.save(proj_fn, zproj) else: zproj = np.load(proj_fn) # DF_F, DF = cse.extract_DF_F(Y.reshape(d1*d2, T), A2, C2) # t0 = time.time() # sys.stdout.write("Ordering components... ") # sys.stdout.flush() # A_or, C_or, srt = cse.order_components(A2, C2) # sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) cm.stop_server() polygons = contour(A2, Y.shape[0], Y.shape[1], thr=roi_iceberg) rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons]) return rois, C2, zproj, S2, Y, YrA