def test_init(self): oc = OpticalChannel('test_name', 'test_source', 'description', 'emission_lambda') self.assertEqual(oc.description, 'description') self.assertEqual(oc.emission_lambda, 'emission_lambda') ip = ImagingPlane('test_imaging_plane', 'test source', oc, 'description', 'device', 'excitation_lambda', 'imaging_rate', 'indicator', 'location', (1, 2, 1, 2, 3), 4.0, 'unit', 'reference_frame') self.assertEqual(ip.optical_channel[0], oc) self.assertEqual(ip.device, 'device') self.assertEqual(ip.excitation_lambda, 'excitation_lambda') self.assertEqual(ip.imaging_rate, 'imaging_rate') self.assertEqual(ip.indicator, 'indicator') self.assertEqual(ip.location, 'location') self.assertEqual(ip.manifold, (1, 2, 1, 2, 3)) self.assertEqual(ip.conversion, 4.0) self.assertEqual(ip.unit, 'unit') self.assertEqual(ip.reference_frame, 'reference_frame') tPS = TwoPhotonSeries('test_tPS', 'a hypothetical source', data=list(), unit='unit', field_of_view=list(), imaging_plane=ip, pmt_gain=1.0, scan_line_rate=2.0, external_file=['external_file'], starting_frame=[1, 2, 3], format='tiff', timestamps=list()) self.assertEqual(tPS.name, 'test_tPS') self.assertEqual(tPS.source, 'a hypothetical source') self.assertEqual(tPS.unit, 'unit') self.assertEqual(tPS.field_of_view, list()) self.assertEqual(tPS.imaging_plane, ip) self.assertEqual(tPS.pmt_gain, 1.0) self.assertEqual(tPS.scan_line_rate, 2.0) self.assertEqual(tPS.external_file, ['external_file']) self.assertEqual(tPS.starting_frame, [1, 2, 3]) self.assertEqual(tPS.format, 'tiff') self.assertEqual(tPS.dimension, [np.nan])
def add_two_photon_series(imaging, nwbfile, metadata, buffer_size=10): """ Auxiliary static method for nwbextractor. Adds two photon series from imaging object as TwoPhotonSeries to nwbfile object. """ metadata = dict_recursive_update(get_default_nwb_metadata(), metadata) metadata = update_dict(metadata, NwbImagingExtractor.get_nwb_metadata(imaging)) # Tests if ElectricalSeries already exists in acquisition nwb_es_names = [ac for ac in nwbfile.acquisition] opts = metadata["Ophys"]["TwoPhotonSeries"][0] if opts["name"] not in nwb_es_names: # retrieve device device = nwbfile.devices[list(nwbfile.devices.keys())[0]] metadata["Ophys"]["ImagingPlane"][0]["optical_channel"] = [ OpticalChannel(**i) for i in metadata["Ophys"]["ImagingPlane"] [0]["optical_channel"] ] metadata["Ophys"]["ImagingPlane"][0] = update_dict( metadata["Ophys"]["ImagingPlane"][0], {"device": device}) imaging_plane = nwbfile.create_imaging_plane( **metadata["Ophys"]["ImagingPlane"][0]) def data_generator(imaging): for i in range(imaging.get_num_frames()): yield imaging.get_frames(frame_idxs=[i]).T data = H5DataIO( DataChunkIterator(data_generator(imaging), buffer_size=buffer_size), compression=True, ) # using internal data. this data will be stored inside the NWB file two_p_series_kwargs = update_dict( metadata["Ophys"]["TwoPhotonSeries"][0], dict(data=data, imaging_plane=imaging_plane), ) ophys_ts = TwoPhotonSeries(**two_p_series_kwargs) nwbfile.add_acquisition(ophys_ts) return nwbfile
def addContainer(self, file): dev1 = file.create_device('dev1', 'dev1 description') oc = OpticalChannel('optchan1', 'a fake OpticalChannel', 3.14) ip = file.create_imaging_plane('imgpln1', oc, 'a fake ImagingPlane', dev1, 6.28, 2.718, 'GFP', 'somewhere in the brain') data = np.ones((3, 3, 3)) timestamps = list(range(10)) fov = [2.0, 2.0, 5.0] tps = TwoPhotonSeries('test_2ps', ip, data, 'image_unit', 'raw', fov, 1.7, 3.4, timestamps=timestamps, dimension=[200, 200]) file.add_acquisition(tps) return tps
def test_init(self): oc = OpticalChannel('test_name', 'description', 500.) self.assertEqual(oc.description, 'description') self.assertEqual(oc.emission_lambda, 500.) device = Device(name='device_name') ip = ImagingPlane('test_imaging_plane', oc, 'description', device, 600., 300., 'indicator', 'location', (50, 100, 3), 4.0, 'unit', 'reference_frame') self.assertEqual(ip.optical_channel[0], oc) self.assertEqual(ip.device, device) self.assertEqual(ip.excitation_lambda, 600.) self.assertEqual(ip.imaging_rate, 300.) self.assertEqual(ip.indicator, 'indicator') self.assertEqual(ip.location, 'location') self.assertEqual(ip.manifold, (50, 100, 3)) self.assertEqual(ip.conversion, 4.0) self.assertEqual(ip.unit, 'unit') self.assertEqual(ip.reference_frame, 'reference_frame') tPS = TwoPhotonSeries('test_tPS', unit='unit', field_of_view=[2., 3.], imaging_plane=ip, pmt_gain=1.0, scan_line_rate=2.0, external_file=['external_file'], starting_frame=[1, 2, 3], format='tiff', timestamps=list()) self.assertEqual(tPS.name, 'test_tPS') self.assertEqual(tPS.unit, 'unit') self.assertEqual(tPS.field_of_view, [2., 3.]) self.assertEqual(tPS.imaging_plane, ip) self.assertEqual(tPS.pmt_gain, 1.0) self.assertEqual(tPS.scan_line_rate, 2.0) self.assertEqual(tPS.external_file, ['external_file']) self.assertEqual(tPS.starting_frame, [1, 2, 3]) self.assertEqual(tPS.format, 'tiff') self.assertIsNone(tPS.dimension)
def test_init(self): ip = create_imaging_plane() tPS = TwoPhotonSeries(name='test_tPS', unit='unit', field_of_view=[2., 3.], imaging_plane=ip, pmt_gain=1.0, scan_line_rate=2.0, external_file=['external_file'], starting_frame=[1, 2, 3], format='tiff', timestamps=list()) self.assertEqual(tPS.name, 'test_tPS') self.assertEqual(tPS.unit, 'unit') self.assertEqual(tPS.field_of_view, [2., 3.]) self.assertEqual(tPS.imaging_plane, ip) self.assertEqual(tPS.pmt_gain, 1.0) self.assertEqual(tPS.scan_line_rate, 2.0) self.assertEqual(tPS.external_file, ['external_file']) self.assertEqual(tPS.starting_frame, [1, 2, 3]) self.assertEqual(tPS.format, 'tiff') self.assertIsNone(tPS.dimension)
def addContainer(self, file): oc = OpticalChannel('optchan1', 'unit test TestImagingPlaneIO', 'a fake OpticalChannel', '3.14') ip = ImagingPlane('imgpln1', 'unit test TestImagingPlaneIO', oc, 'a fake ImagingPlane', 'imaging_device_1', '6.28', '2.718', 'GFP', 'somewhere in the brain') file.set_imaging_plane(ip) data = list(zip(range(10), range(10, 20))) timestamps = list(range(10)) fov = [2.0, 2.0, 5.0] tps = TwoPhotonSeries('test_2ps', 'unit test TestTwoPhotonSeries', data, ip, 'image_unit', 'raw', fov, 1.7, 3.4, timestamps=timestamps, dimension=[2]) file.add_acquisition(tps)
def setUpClass(self): device = Device("imaging_device_1") optical_channel = OpticalChannel("my_optchan", "description", 500.0) self.imaging_plane = ImagingPlane( name="imgpln1", optical_channel=optical_channel, description="a fake ImagingPlane", device=device, excitation_lambda=600.0, imaging_rate=300.0, indicator="GFP", location="somewhere in the brain", reference_frame="unknown", origin_coords=[10, 20], origin_coords_unit="millimeters", grid_spacing=[0.001, 0.001], grid_spacing_unit="millimeters", ) self.image_series = TwoPhotonSeries( name="test_image_series", data=np.random.randn(100, 5, 5), imaging_plane=self.imaging_plane, starting_frame=[0], rate=1.0, unit="n.a", ) self.img_seg = ImageSegmentation() self.ps2 = self.img_seg.create_plane_segmentation( "output from segmenting my favorite imaging plane", self.imaging_plane, "2d_plane_seg", self.image_series, ) self.ps2.add_column("type", "desc") self.ps2.add_column("type2", "desc") w, h = 3, 3 img_mask1 = np.zeros((w, h)) img_mask1[0, 0] = 1.1 img_mask1[1, 1] = 1.2 img_mask1[2, 2] = 1.3 self.ps2.add_roi(image_mask=img_mask1, type=1, type2=0) img_mask2 = np.zeros((w, h)) img_mask2[0, 0] = 2.1 img_mask2[1, 1] = 2.2 self.ps2.add_roi(image_mask=img_mask2, type=1, type2=1) img_mask2 = np.zeros((w, h)) img_mask2[0, 0] = 9.1 img_mask2[1, 1] = 10.2 self.ps2.add_roi(image_mask=img_mask2, type=2, type2=0) img_mask2 = np.zeros((w, h)) img_mask2[0, 0] = 3.5 img_mask2[1, 1] = 5.6 self.ps2.add_roi(image_mask=img_mask2, type=2, type2=1) fl = Fluorescence() rt_region = self.ps2.create_roi_table_region("the first of two ROIs", region=[0, 1, 2, 3]) rois_shape = 5 data = np.arange(10 * rois_shape).reshape([10, -1], order='F') timestamps = np.array( [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]) rrs = fl.create_roi_response_series(name="my_rrs", data=data, rois=rt_region, unit="lumens", timestamps=timestamps) self.df_over_f = DfOverF(rrs)
device='imaging_device_1', excitation_lambda='unknown', imaging_rate='unknown', indicator='GFP', location='unknown', manifold=np.array([]), conversion=1.0, unit='um', reference_frame='A frame to refer to') for channel_name, imaging_data in zip(channel_names, all_imaging_data): image_series = TwoPhotonSeries(name='image', source='Ca2+ imaging example', dimension=[2], data=imaging_data, imaging_plane=imaging_plane, starting_frame=[0], timestamps=[1, 2, 3], scan_line_rate=np.nan, pmt_gain=np.nan) nwbfile.add_acquisition(image_series) out_fname = 'sebi_data.nwb' print('writing NWB file...', end='', flush=True) with NWBHDF5IO(out_fname, mode='w') as io: io.write(nwbfile, cache_spec=False) print('done.') print('testing read...', end='', flush=True) # test read with NWBHDF5IO(out_fname, mode='r') as io:
device=device, excitation_lambda=np.nan, imaging_rate=np.nan, indicator='GFP', location='unknown', manifold=np.array([]), conversion=1.0, unit='um', reference_frame='A frame to refer to') for channel_name, imaging_data in zip(channel_names, all_imaging_data): image_series = TwoPhotonSeries(name='TwoPhotonSeries' + channel_name.decode(), dimension=[2], data=H5DataIO(imaging_data, compression='gzip'), imaging_plane=imaging_plane, starting_frame=[0], timestamps=[1, 2, 3], scan_line_rate=np.nan, pmt_gain=np.nan) nwbfile.add_acquisition(image_series) if SHORTEN: out_fname = fpath + '_stub.nwb' else: out_fname = fpath + '.nwb' print('writing NWB file...', end='', flush=True) with NWBHDF5IO(out_fname, mode='w') as io: io.write(nwbfile) print('done.')
def save_nwb(ops1): if NWB and not ops1[0]['mesoscan']: if len(ops1) > 1: multiplane = True else: multiplane = False ops = ops1[0] ### INITIALIZE NWB FILE nwbfile = NWBFile( session_description='suite2p_proc', identifier=ops['data_path'][0], session_start_time=(ops['date_proc'] if 'date_proc' in ops else datetime.datetime.now())) print(nwbfile) device = nwbfile.create_device( name='Microscope', description='My two-photon microscope', manufacturer='The best microscope manufacturer') optical_channel = OpticalChannel(name='OpticalChannel', description='an optical channel', emission_lambda=500.) imaging_plane = nwbfile.create_imaging_plane( name='ImagingPlane', optical_channel=optical_channel, imaging_rate=ops['fs'], description='standard', device=device, excitation_lambda=600., indicator='GCaMP', location='V1', grid_spacing=([2, 2, 30] if multiplane else [2, 2]), grid_spacing_unit='microns') # link to external data image_series = TwoPhotonSeries( name='TwoPhotonSeries', dimension=[ops['Ly'], ops['Lx']], external_file=(ops['filelist'] if 'filelist' in ops else ['']), imaging_plane=imaging_plane, starting_frame=[0], format='external', starting_time=0.0, rate=ops['fs'] * ops['nplanes']) nwbfile.add_acquisition(image_series) # processing img_seg = ImageSegmentation() ps = img_seg.create_plane_segmentation(name='PlaneSegmentation', description='suite2p output', imaging_plane=imaging_plane, reference_images=image_series) ophys_module = nwbfile.create_processing_module( name='ophys', description='optical physiology processed data') ophys_module.add(img_seg) file_strs = ['F.npy', 'Fneu.npy', 'spks.npy'] traces = [] ncells_all = 0 for iplane, ops in enumerate(ops1): if iplane == 0: iscell = np.load(os.path.join(ops['save_path'], 'iscell.npy')) for fstr in file_strs: traces.append(np.load(os.path.join(ops['save_path'], fstr))) else: iscell = np.append(iscell, np.load( os.path.join(ops['save_path'], 'iscell.npy')), axis=0) for i, fstr in enumerate(file_strs): traces[i] = np.append( traces[i], np.load(os.path.join(ops['save_path'], fstr)), axis=0) stat = np.load(os.path.join(ops['save_path'], 'stat.npy'), allow_pickle=True) ncells = len(stat) for n in range(ncells): if multiplane: pixel_mask = np.array([ stat[n]['ypix'], stat[n]['xpix'], iplane * np.ones(stat[n]['npix']), stat[n]['lam'] ]) ps.add_roi(voxel_mask=pixel_mask.T) else: pixel_mask = np.array( [stat[n]['ypix'], stat[n]['xpix'], stat[n]['lam']]) ps.add_roi(pixel_mask=pixel_mask.T) ncells_all += ncells ps.add_column('iscell', 'two columns - iscell & probcell', iscell) rt_region = ps.create_roi_table_region(region=list( np.arange(0, ncells_all)), description='all ROIs') # FLUORESCENCE (all are required) file_strs = ['F.npy', 'Fneu.npy', 'spks.npy'] name_strs = ['Fluorescence', 'Neuropil', 'Deconvolved'] for i, (fstr, nstr) in enumerate(zip(file_strs, name_strs)): roi_resp_series = RoiResponseSeries(name=nstr, data=traces[i], rois=rt_region, unit='lumens', rate=ops['fs']) fl = Fluorescence(roi_response_series=roi_resp_series, name=nstr) ophys_module.add(fl) # BACKGROUNDS # (meanImg, Vcorr and max_proj are REQUIRED) bg_strs = ['meanImg', 'Vcorr', 'max_proj', 'meanImg_chan2'] nplanes = ops['nplanes'] for iplane in range(nplanes): images = Images('Backgrounds_%d' % iplane) for bstr in bg_strs: if bstr in ops: if bstr == 'Vcorr' or bstr == 'max_proj': img = np.zeros((ops['Ly'], ops['Lx']), np.float32) img[ops['yrange'][0]:ops['yrange'][-1], ops['xrange'][0]:ops['xrange'][-1]] = ops[bstr] else: img = ops[bstr] images.add_image(GrayscaleImage(name=bstr, data=img)) ophys_module.add(images) with NWBHDF5IO(os.path.join(ops['save_path0'], 'suite2p', 'ophys.nwb'), 'w') as fio: fio.write(nwbfile) else: print('pip install pynwb OR don"t use mesoscope recording')
def setUp(self): start_time = datetime(2017, 4, 3, 11, 0, 0) create_date = datetime(2017, 4, 15, 12, 0, 0) # create your NWBFile object nwbfile = NWBFile('PyNWB Sample File', 'A simple NWB file', 'NWB_test', start_time, file_create_date=create_date) # create acquisition metadata optical_channel = OpticalChannel('test_optical_channel', 'optical channel source', 'optical channel description', 3.14) imaging_plane = nwbfile.create_imaging_plane('test_imaging_plane', 'ophys integration tests', optical_channel, 'imaging plane description', 'imaging_device_1', 6.28, '2.718', 'GFP', 'somewhere in the brain', (1, 2, 1, 2, 3), 4.0, 'manifold unit', 'A frame to refer to') # create acquisition data image_series = TwoPhotonSeries(name='test_iS', source='a hypothetical source', dimension=[2], external_file=['images.tiff'], imaging_plane=imaging_plane, starting_frame=[1, 2, 3], format='tiff', timestamps=list()) nwbfile.add_acquisition(image_series) mod = nwbfile.create_processing_module('img_seg_example', 'ophys demo', 'an example of writing Ca2+ imaging data') img_seg = ImageSegmentation('a toy image segmentation container') mod.add_data_interface(img_seg) ps = img_seg.create_plane_segmentation('integration test PlaneSegmentation', 'plane segmentation description', imaging_plane, 'test_plane_seg_name', image_series) # add two ROIs # - first argument is the pixel mask i.e. a list of pixels and their weights # - second argument is the image mask w, h = 3, 3 pix_mask1 = [(0, 0, 1.1), (1, 1, 1.2), (2, 2, 1.3)] img_mask1 = [[0.0 for x in range(w)] for y in range(h)] img_mask1[0][0] = 1.1 img_mask1[1][1] = 1.2 img_mask1[2][2] = 1.3 ps.add_roi('1234', pix_mask1, img_mask1) pix_mask2 = [(0, 0, 2.1), (1, 1, 2.2)] img_mask2 = [[0.0 for x in range(w)] for y in range(h)] img_mask2[0][0] = 2.1 img_mask2[1][1] = 2.2 ps.add_roi('5678', pix_mask2, img_mask2) # add a Fluorescence container fl = Fluorescence('a toy fluorescence container') mod.add_data_interface(fl) # get an ROI table region i.e. a subset of ROIs to create a RoiResponseSeries from rt_region = ps.create_roi_table_region('the first of two ROIs', region=[0]) # make some fake timeseries data data = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] timestamps = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] rrs = fl.create_roi_response_series('test_roi_response_series', 'RoiResponseSeries integration test', data, 'lumens', rt_region, timestamps=timestamps) # write data nwb_path = './test_data/nwb_test_file.nwb' with NWBHDF5IO(nwb_path, 'w') as io: io.write(nwbfile)
def add_two_photon_series(imaging, nwbfile, metadata, num_chunks=10): """ Auxiliary static method for nwbextractor. Adds two photon series from imaging object as TwoPhotonSeries to nwbfile object. """ if 'Ophys' not in metadata: metadata['Ophys'] = {} if 'TwoPthotonSeries' not in metadata['Ophys']: metadata['Ophys']['TwoPhotonSeries'] = [{ 'name': 'TwoPhotonSeries', 'description': 'optical_series_description' }] # Tests if ElectricalSeries already exists in acquisition nwb_es_names = [ac for ac in nwbfile.acquisition] opts = metadata['Ophys']['TwoPhotonSeries'][0] if opts['name'] not in nwb_es_names: # retrieve device device = nwbfile.devices[list(nwbfile.devices.keys())[0]] # create optical channel if 'OpticalChannel' not in metadata['Ophys']: metadata['Ophys']['OpticalChannel'] = [{ 'name': 'OpticalChannel', 'description': 'no description', 'emission_lambda': np.nan }] optical_channel = OpticalChannel( **metadata['Ophys']['OpticalChannel'][0]) # sampling rate rate = float(imaging.get_sampling_frequency()) if 'ImagingPlane' not in metadata['Ophys']: metadata['Ophys']['ImagingPlane'] = [{ 'name': 'ImagingPlane', 'description': 'no description', 'excitation_lambda': np.nan, 'indicator': 'unknown', 'location': 'unknown' }] imaging_meta = { 'optical_channel': optical_channel, 'imaging_rate': rate, 'device': device } metadata['Ophys']['ImagingPlane'][0] = update_dict( metadata['Ophys']['ImagingPlane'][0], imaging_meta) imaging_plane = nwbfile.create_imaging_plane( **metadata['Ophys']['ImagingPlane'][0]) def data_generator(imaging, num_chunks): num_frames = imaging.get_num_frames() # chunk size is not None chunk_size = num_frames // num_chunks if num_frames % chunk_size > 0: num_chunks += 1 for i in range(num_chunks): video = imaging.get_video(start_frame=i * chunk_size, end_frame=min( (i + 1) * chunk_size, num_frames)) data = np.squeeze(video) yield data data = H5DataIO(DataChunkIterator( data_generator(imaging, num_chunks)), compression=True) acquisition_name = opts['name'] # using internal data. this data will be stored inside the NWB file ophys_ts = TwoPhotonSeries( name=acquisition_name, data=data, imaging_plane=imaging_plane, rate=rate, unit='normalized amplitude', comments='Generated from RoiInterface::NwbImagingExtractor', description='no description') nwbfile.add_acquisition(ophys_ts) return nwbfile
def add_imaging(nwbfile, expt, z_spacing=25., device_name='2P Microscope', location='CA1', indicator='GCaMP6f', excitation_lambda=920., data_root=None, stub=False): color_dict = {'Ch1': 'Red', 'Ch2': 'Green'} # Emissions for mCherry and GCaMP # TODO make this more flexible emission = {'Ch1': 640., 'Ch2': 530.} ch_names = ['Ch1', 'Ch2'] optical_channels = [] for ch_name in ch_names: optical_channel = OpticalChannel(name=ch_name, description=color_dict[ch_name], emission_lambda=emission[ch_name]) optical_channels.append(optical_channel) h5_path = glob(os.path.join(data_root, '*.h5'))[0] pv_xml = os.path.join(data_root, os.path.basename(data_root) + '.xml') pv_version = get_prairieview_version(pv_xml) [y_um, x_um] = get_element_size_um(pv_xml, pv_version)[-2:] elem_size_um = [z_spacing, y_um, x_um] # TODO allow for flexibility in setting device, excitation, indicator, location # TODO nwb-schema issue #151 needs to be resolved so we can actually use imaging data size device = nwbfile.create_device(device_name) imaging_plane = nwbfile.create_imaging_plane( name='Imaging Data', optical_channel=optical_channels, description='imaging data for both channels', device=device, excitation_lambda=excitation_lambda, imaging_rate=1 / expt.frame_period(), indicator=indicator, location=location, conversion=1.0, # Should actually be elem_size_um manifold=np.ones((2, 2, 2, 3)), reference_frame='reference_frame', unit='um') f = h5py.File(h5_path, 'r') imaging_data = f['imaging'] channel_names = f['imaging'].attrs['channel_names'] for c, channel_name in enumerate(channel_names): if not stub: data_in = H5DataIO(DataChunkIterator(tqdm( (np.swapaxes(data[..., c], 0, 2) for data in imaging_data), total=imaging_data.shape[0]), buffer_size=5000), compression='gzip') else: data_in = np.ones((10, 10, 10)) # use for dev testing for speed # TODO parse env file to add power and pmt gain? image_series = TwoPhotonSeries(name='2p_Series_' + channel_name, dimension=expt.frame_shape()[:-1], data=data_in, imaging_plane=imaging_plane, rate=1 / expt.frame_period(), starting_time=0., description=channel_name) nwbfile.add_acquisition(image_series)
imaging_plane = nwbfile.create_imaging_plane(name='Culture', optical_channel=optical_channel_2, imaging_rate=400., description='Voltage imaging', device=device_2, excitation_lambda=000., indicator='Quasar-2', location='') # grid_spacing=[.01, .01], ## how much µm is 1 pixel ? # grid_spacing_unit='µm') # using internal data. this data will be stored inside the NWB file as acquired data that is not-modifiable # can add multiple series if wanted image_series1 = TwoPhotonSeries(name='fluorescence data', data=data, imaging_plane=imaging_plane, rate=400.0, unit='pixel intensity') nwbfile.add_acquisition(image_series1) ## Create a ProcessingModule to store the future processed data ophys_module = nwbfile.create_processing_module( name='ophys', description='optical physiology processed data') ## ImageSegmentation object and add the ImageSegmentation to the ophy module. img_seg = ImageSegmentation( ) ## imagesegmentation can contain multiple planesegmentation ophys_module.add(img_seg) ## Create PlaneSegmentation tables within the targetted imaging_plane ps = img_seg.create_plane_segmentation(
#################### # Adding two-photon image data # ---------------------------- # # Now that you have your :py:class:`~pynwb.ophys.ImagingPlane`, you can create a # :py:class:`~pynwb.ophys.TwoPhotonSeries` - the class representing two photon imaging data. # # From here you have two options. The first option is to supply the image data to PyNWB, using the `data` argument. # The other option is the provide a path the images. These two options have trade-offs, so it is worth spending time # considering how you want to store this data [#]_. image_series = TwoPhotonSeries(name='test_iS', source='Ca2+ imaging example', dimension=[2], external_file=['images.tiff'], imaging_plane=imaging_plane, starting_frame=[0], format='tiff', timestamps=list()) nwbfile.add_acquisition(image_series) #################### # Storing image segmentation output # --------------------------------- # # Now that the raw data is stored, you can add the image segmentation results. This is done with the # :py:class:`~pynwb.ophys.ImageSegmentation` data interface. This class has the ability to store segmentation # from one or more imaging planes; hence the :py:class:`~pynwb.ophys.PlaneSegmentation` class. mod = nwbfile.create_processing_module('my_ca_imaging_module', 'Ca2+ imaging example', 'example data module') img_seg = ImageSegmentation('Ca2+ imaging example')
def add_ophys_processing_from_suite2p(save_folder, nwbfile, CaImaging_timestamps, device=None, optical_channel=None, imaging_plane=None, image_series=None): """ adapted from suite2p/suite2p/io/nwb.py "save_nwb" function """ plane_folders = natsorted([ f.path for f in os.scandir(save_folder) if f.is_dir() and f.name[:5] == 'plane' ]) ops1 = [ np.load(os.path.join(f, 'ops.npy'), allow_pickle=True).item() for f in plane_folders ] if len(ops1) > 1: multiplane = True else: multiplane = False ops = ops1[0] if device is None: device = nwbfile.create_device( name='Microscope', description='My two-photon microscope', manufacturer='The best microscope manufacturer') if optical_channel is None: optical_channel = OpticalChannel(name='OpticalChannel', description='an optical channel', emission_lambda=500.) if imaging_plane is None: imaging_plane = nwbfile.create_imaging_plane( name='ImagingPlane', optical_channel=optical_channel, imaging_rate=ops['fs'], description='standard', device=device, excitation_lambda=600., indicator='GCaMP', location='V1', grid_spacing=([2, 2, 30] if multiplane else [2, 2]), grid_spacing_unit='microns') if image_series is None: # link to external data image_series = TwoPhotonSeries( name='TwoPhotonSeries', dimension=[ops['Ly'], ops['Lx']], external_file=(ops['filelist'] if 'filelist' in ops else ['']), imaging_plane=imaging_plane, starting_frame=[0], format='external', starting_time=0.0, rate=ops['fs'] * ops['nplanes']) nwbfile.add_acquisition(image_series) # otherwise, were added # processing img_seg = ImageSegmentation() ps = img_seg.create_plane_segmentation(name='PlaneSegmentation', description='suite2p output', imaging_plane=imaging_plane, reference_images=image_series) ophys_module = nwbfile.create_processing_module( name='ophys', description='optical physiology processed data') ophys_module.add(img_seg) file_strs = ['F.npy', 'Fneu.npy', 'spks.npy'] traces = [] ncells_all = 0 for iplane, ops in enumerate(ops1): if iplane == 0: iscell = np.load( os.path.join(save_folder, 'plane%i' % iplane, 'iscell.npy')) for fstr in file_strs: traces.append( np.load(os.path.join(save_folder, 'plane%i' % iplane, fstr))) else: iscell = np.append(iscell, np.load( os.path.join(save_folder, 'plane%i' % iplane, 'iscell.npy')), axis=0) for i, fstr in enumerate(file_strs): traces[i] = np.append( traces[i], np.load(os.path.join(save_folder, 'plane%i' % iplane, fstr)), axis=0) stat = np.load(os.path.join(save_folder, 'plane%i' % iplane, 'stat.npy'), allow_pickle=True) ncells = len(stat) for n in range(ncells): if multiplane: pixel_mask = np.array([ stat[n]['ypix'], stat[n]['xpix'], iplane * np.ones(stat[n]['npix']), stat[n]['lam'] ]) ps.add_roi(voxel_mask=pixel_mask.T) else: pixel_mask = np.array( [stat[n]['ypix'], stat[n]['xpix'], stat[n]['lam']]) ps.add_roi(pixel_mask=pixel_mask.T) ncells_all += ncells ps.add_column('iscell', 'two columns - iscell & probcell', iscell) rt_region = ps.create_roi_table_region(region=list(np.arange( 0, ncells_all)), description='all ROIs') # FLUORESCENCE (all are required) file_strs = ['F.npy', 'Fneu.npy', 'spks.npy'] name_strs = ['Fluorescence', 'Neuropil', 'Deconvolved'] for i, (fstr, nstr) in enumerate(zip(file_strs, name_strs)): roi_resp_series = RoiResponseSeries( name=nstr, data=traces[i], rois=rt_region, unit='lumens', timestamps=CaImaging_timestamps ) # CRITICAL TO HAVE IT HERE FOR RE-ALIGNEMENT fl = Fluorescence(roi_response_series=roi_resp_series, name=nstr) ophys_module.add(fl) # BACKGROUNDS # (meanImg, Vcorr and max_proj are REQUIRED) bg_strs = ['meanImg', 'meanImgE', 'Vcorr', 'max_proj', 'meanImg_chan2'] nplanes = ops['nplanes'] for iplane in range(nplanes): images = Images('Backgrounds_%d' % iplane) for bstr in bg_strs: if bstr in ops: if bstr == 'Vcorr' or bstr == 'max_proj': img = np.zeros((ops['Ly'], ops['Lx']), np.float32) img[ops['yrange'][0]:ops['yrange'][-1], ops['xrange'][0]:ops['xrange'][-1]] = ops[bstr] else: img = ops[bstr] images.add_image(GrayscaleImage(name=bstr, data=img)) ophys_module.add(images)
def convert(self, **kwargs): """Convert the data and add to the nwb_file Args: **kwargs: arbitrary arguments """ super().convert(**kwargs) # ### setting parameters #### formats_implemented = ["external", "tiff"] if not kwargs.get("format"): raise Exception(f"'format' argument should be pass to convert function in class {self.__class__.__name__}") elif kwargs["format"] not in formats_implemented: raise Exception(f"'format' argument should have one of these values {formats_implemented} " f"for the convert function in class {self.__class__.__name__}") movie_format = kwargs["format"] if not kwargs.get("motion_corrected_file_name"): raise Exception(f"'motion_corrected_file_name' attribute should be pass to convert " f"function in class {self.__class__.__name__}") motion_corrected_file_name = kwargs["motion_corrected_file_name"] if "original_movie_file_name" in kwargs: original_movie_file_name = kwargs["original_movie_file_name"] else: original_movie_file_name = None if "xy_translation_file_name" in kwargs: xy_translation_file_name = kwargs["xy_translation_file_name"] else: xy_translation_file_name = None # Open YAML file with metadata if existing then dump all data in a dict if ("yaml_file_name" in kwargs) and kwargs["yaml_file_name"] is not None: with open(kwargs["yaml_file_name"], 'r') as stream: yaml_data = yaml.safe_load(stream) else: raise Exception(f"'yaml_file_name' attribute should be pass to convert " f"function in class {self.__class__.__name__}") # a calcium imaging rate has to be given, either trought the yaml file, either as argument # self.ci_sampling_rate can be obtained by the abf_converter if "imaging_rate" in yaml_data: self.ci_sampling_rate = yaml_data["imaging_rate"] elif "ci_sampling_rate" in kwargs: self.ci_sampling_rate = kwargs["ci_sampling_rate"] else: raise Exception(f"No 'imaging_rate' provided for movie {motion_corrected_file_name} in the yaml file " f"{kwargs['yaml_file_name']} or throught argument 'ci_sampling_rate' to function convert() " f"of the class {self.__class__.__name__}") if "indicator" in yaml_data: indicator = yaml_data["indicator"] else: raise Exception(f"No 'indicator' provided for movie {motion_corrected_file_name} in the yaml file " f"{kwargs['yaml_file_name']}") if "excitation_lambda" in yaml_data: excitation_lambda = yaml_data["excitation_lambda"] else: raise Exception(f"No 'excitation_lambda' provided for movie {motion_corrected_file_name} in the yaml file " f"{kwargs['yaml_file_name']}") if "emission_lambda" in yaml_data: emission_lambda = yaml_data["emission_lambda"] else: raise Exception(f"No 'emission_lambda' provided for movie {motion_corrected_file_name} in the yaml file " f"{kwargs['yaml_file_name']}") if "image_plane_location" in yaml_data: image_plane_location = yaml_data["image_plane_location"] else: raise Exception( f"No 'image_plane_location' provided for movie {motion_corrected_file_name} in the yaml file " f"{kwargs['yaml_file_name']}") try : if 'ci_recording_on_pause' in self.nwb_file.intervals: pause_intervals = self.nwb_file.intervals['ci_recording_on_pause'] pause_intervals_df = pause_intervals.to_dataframe() start_times = pause_intervals_df.loc[:, "start_time"] stop_times = pause_intervals_df.loc[:, "stop_time"] try: ci_frames_time_series = self.nwb_file.get_acquisition("ci_frames") ci_frames = np.where(ci_frames_time_series.data)[0] ci_frames_timestamps = ci_frames_time_series.timestamps[ci_frames] for i, start_time in enumerate(start_times): frame_index = np.searchsorted(a=ci_frames_timestamps, v=start_time) n_frames_to_add = (stop_times[i] - start_time) * self.ci_sampling_rate self.frames_to_add[frame_index] = int(n_frames_to_add) except KeyError: pass except AttributeError: pass # ### end setting parameters #### device = Device('2P_device') self.nwb_file.add_device(device) optical_channel = OpticalChannel('my_optchan', 'description', emission_lambda) imaging_plane = self.nwb_file.create_imaging_plane(name='my_imgpln', optical_channel=optical_channel, description='a very interesting part of the brain', device=device, excitation_lambda=excitation_lambda, imaging_rate=float(self.ci_sampling_rate), indicator=indicator, location=image_plane_location) if movie_format != "external": tiff_movie = self.load_tiff_movie_in_memory(motion_corrected_file_name) dim_y, dim_x = tiff_movie.shape[1:] n_frames = tiff_movie.shape[0] motion_corrected_img_series = TwoPhotonSeries(name='motion_corrected_ci_movie', dimension=[dim_x, dim_y], data=tiff_movie, imaging_plane=imaging_plane, starting_frame=[0], format=movie_format, rate=self.ci_sampling_rate) if original_movie_file_name is not None: original_tiff_movie = self.load_tiff_movie_in_memory(original_movie_file_name) dim_y, dim_x = original_tiff_movie.shape[1:] original_img_series = TwoPhotonSeries(name='original_ci_movie', dimension=[dim_x, dim_y], data=original_tiff_movie, imaging_plane=imaging_plane, starting_frame=[0], format=movie_format, rate=float(self.ci_sampling_rate)) else: im = PIL.Image.open(motion_corrected_file_name) n_frames = len(list(ImageSequence.Iterator(im))) dim_y, dim_x = np.array(im).shape motion_corrected_img_series = TwoPhotonSeries(name='motion_corrected_ci_movie', dimension=[dim_x, dim_y], external_file=[motion_corrected_file_name], imaging_plane=imaging_plane, starting_frame=[0], format=movie_format, rate=float(self.ci_sampling_rate)) if original_movie_file_name is not None: im = PIL.Image.open(original_movie_file_name) dim_y, dim_x = np.array(im).shape original_img_series = TwoPhotonSeries(name='original_ci_movie', dimension=[dim_x, dim_y], external_file=[original_movie_file_name], imaging_plane=imaging_plane, starting_frame=[0], format=movie_format, rate=float(self.ci_sampling_rate)) self.nwb_file.add_acquisition(motion_corrected_img_series) if original_movie_file_name is not None: self.nwb_file.add_acquisition(original_img_series) if xy_translation_file_name is not None: if xy_translation_file_name.endswith(".mat"): mvt_x_y = hdf5storage.loadmat(os.path.join(xy_translation_file_name)) x_shifts = mvt_x_y['xshifts'][0] y_shifts = mvt_x_y['yshifts'][0] elif xy_translation_file_name.endswith(".npy"): ops = np.load(os.path.join(xy_translation_file_name)) ops = ops.item() x_shifts = ops['xoff'] y_shifts = ops['yoff'] xy_translation = np.zeros((n_frames, 2), dtype="int16") frame_index = 0 for frame in np.arange(len(x_shifts)): xy_translation[frame_index, 0] = x_shifts[frame] xy_translation[frame_index, 1] = y_shifts[frame] # adding frames is necessary, in case the movie would be a concatenation of movie for exemple if frame in self.frames_to_add: frame_index += self.frames_to_add[frame] xy_translation_time_series = TimeSeries(name="xy_translation", data=xy_translation) corrected_image_stack = CorrectedImageStack(name="CorrectedImageStack", corrected=motion_corrected_img_series, original=original_img_series, xy_translation=xy_translation_time_series) self.nwb_file.add_acquisition(corrected_image_stack)
def setUp(self): nwbfile = NWBFile( 'my first synthetic recording', 'EXAMPLE_ID', datetime.now(tzlocal()), experimenter='Dr. Bilbo Baggins', lab='Bag End Laboratory', institution='University of Middle Earth at the Shire', experiment_description=('I went on an adventure with thirteen ' 'dwarves to reclaim vast treasures.'), session_id='LONELYMTN') device = Device('imaging_device_1') nwbfile.add_device(device) optical_channel = OpticalChannel('my_optchan', 'description', 500.) imaging_plane = nwbfile.create_imaging_plane( 'my_imgpln', optical_channel, 'a very interesting part of the brain', device, 600., 300., 'GFP', 'my favorite brain location', np.ones((5, 5, 3)), 4.0, 'manifold unit', 'A frame to refer to') self.image_series = TwoPhotonSeries(name='test_iS', dimension=[2], data=np.random.rand(10, 5, 5, 3), external_file=['images.tiff'], imaging_plane=imaging_plane, starting_frame=[0], format='tiff', starting_time=0.0, rate=1.0) nwbfile.add_acquisition(self.image_series) mod = nwbfile.create_processing_module( 'ophys', 'contains optical physiology processed data') img_seg = ImageSegmentation() mod.add(img_seg) ps = img_seg.create_plane_segmentation( 'output from segmenting my favorite imaging plane', imaging_plane, 'my_planeseg', self.image_series) w, h = 3, 3 pix_mask1 = [(0, 0, 1.1), (1, 1, 1.2), (2, 2, 1.3)] vox_mask1 = [(0, 0, 0, 1.1), (1, 1, 1, 1.2), (2, 2, 2, 1.3)] img_mask1 = [[0.0 for x in range(w)] for y in range(h)] img_mask1[0][0] = 1.1 img_mask1[1][1] = 1.2 img_mask1[2][2] = 1.3 ps.add_roi(pixel_mask=pix_mask1, image_mask=img_mask1, voxel_mask=vox_mask1) pix_mask2 = [(0, 0, 2.1), (1, 1, 2.2)] vox_mask2 = [(0, 0, 0, 2.1), (1, 1, 1, 2.2)] img_mask2 = [[0.0 for x in range(w)] for y in range(h)] img_mask2[0][0] = 2.1 img_mask2[1][1] = 2.2 ps.add_roi(pixel_mask=pix_mask2, image_mask=img_mask2, voxel_mask=vox_mask2) fl = Fluorescence() mod.add(fl) rt_region = ps.create_roi_table_region('the first of two ROIs', region=[0]) data = np.array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.]).reshape(10, 1) timestamps = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] rrs = fl.create_roi_response_series('my_rrs', data, rt_region, unit='lumens', timestamps=timestamps) self.df_over_f = DfOverF(rrs)
#################### # Adding two-photon image data # ---------------------------- # # Now that you have your :py:class:`~pynwb.ophys.ImagingPlane`, you can create a # :py:class:`~pynwb.ophys.TwoPhotonSeries` - the class representing two photon imaging data. # # From here you have two options. The first option is to supply the image data to PyNWB, using the `data` argument. # The other option is the provide a path the images. These two options have trade-offs, so it is worth spending time # considering how you want to store this data [#]_. image_series = TwoPhotonSeries(name='test_iS', dimension=[2], external_file=['images.tiff'], imaging_plane=imaging_plane, starting_frame=[0], format='tiff', starting_time=0.0, rate=1.0) nwbfile.add_acquisition(image_series) #################### # Storing image segmentation output # --------------------------------- # # Now that the raw data is stored, you can add the image segmentation results. This is done with the # :py:class:`~pynwb.ophys.ImageSegmentation` data interface. This class has the ability to store segmentation # from one or more imaging planes; hence the :py:class:`~pynwb.ophys.PlaneSegmentation` class. mod = nwbfile.create_processing_module('my_ca_imaging_module', 'example data module')
optical_channel = OpticalChannel('my channel', 'description', np.nan) imaging_plane = nwbfile.create_imaging_plane('my_imgpln', optical_channel, 'description', 'imaging_device_1', 600., '2.718', 'GFP', 'my favorite brain location', [], 4.0, 'manifold unit', 'A frame to refer to') images_path = os.path.join(base_dir, 'm655_D11_S1.hdf5') with File(images_path, 'r') as file: image_series = TwoPhotonSeries(name='test_iS', dimension=[2], data=file['Data']['Images'][:], imaging_plane=imaging_plane, starting_frame=[0], starting_time=0.0, rate=5.0, scan_line_rate=np.nan, pmt_gain=np.nan) nwbfile.add_acquisition(image_series) mod = nwbfile.create_processing_module('rois', 'example data module') img_seg = ImageSegmentation() ps = img_seg.create_plane_segmentation('Ca2+ imaging example', imaging_plane, 'my_planeseg', image_series) mod.add_data_interface(img_seg) for i, img_mask in enumerate(zip(mat_data['cellImages'])): pixel_mask = np.array(np.where(img_mask)).T ps.add_roi(str(i), pixel_mask, img_mask)
# :width: 800 # :alt: two-photon series UML diagram # :align: center # # .. only:: latex # # .. image:: ../../_static/TwoPhotonSeries.png # :width: 800 # :alt: two-photon series UML diagram # :align: center # # using internal data. this data will be stored inside the NWB file image_series1 = TwoPhotonSeries(name='TwoPhotonSeries1', data=np.ones((1000, 100, 100)), imaging_plane=imaging_plane, rate=1.0, unit='normalized amplitude') # using external data. only the file paths will be stored inside the NWB file image_series2 = TwoPhotonSeries(name='TwoPhotonSeries2', dimension=[100, 100], external_file=['images.tiff'], imaging_plane=imaging_plane, starting_frame=[0], format='external', starting_time=0.0, rate=1.0) #################### # Since these two-photon data are raw, acquired data, we will add the
def conversion_function(source_paths, f_nwb, metadata, add_raw=False, add_processed=True, add_behavior=True, plot_rois=False): """ Copy data stored in a set of .npz files to a single NWB file. Parameters ---------- source_paths : dict Dictionary with paths to source files/directories. e.g.: {'raw_data': {'type': 'file', 'path': ''}, 'raw_info': {'type': 'file', 'path': ''} 'processed_data': {'type': 'file', 'path': ''}, 'sparse_matrix': {'type': 'file', 'path': ''}, 'ref_image',: {'type': 'file', 'path': ''}} f_nwb : str Path to output NWB file, e.g. 'my_file.nwb'. metadata : dict Metadata dictionary add_raw : bool Whether to convert raw data or not. add_processed : bool Whether to convert processed data or not. add_behavior : bool Whether to convert behavior data or not. plot_rois : bool Plot ROIs """ # Source files file_raw = None file_info = None file_processed = None file_sparse_matrix = None file_reference_image = None for k, v in source_paths.items(): if source_paths[k]['path'] != '': fname = source_paths[k]['path'] if k == 'raw_data': file_raw = h5py.File(fname, 'r') if k == 'raw_info': file_info = scipy.io.loadmat(fname, struct_as_record=False, squeeze_me=True) if k == 'processed_data': file_processed = np.load(fname) if k == 'sparse_matrix': file_sparse_matrix = np.load(fname) if k == 'ref_image': file_reference_image = np.load(fname) # Initialize a NWB object nwb = NWBFile(**metadata['NWBFile']) # Create and add device device = Device(name=metadata['Ophys']['Device'][0]['name']) nwb.add_device(device) # Creates one Imaging Plane for each channel fs = 1. / (file_processed['time'][0][1] - file_processed['time'][0][0]) for meta_ip in metadata['Ophys']['ImagingPlane']: # Optical channel opt_ch = OpticalChannel( name=meta_ip['optical_channel'][0]['name'], description=meta_ip['optical_channel'][0]['description'], emission_lambda=meta_ip['optical_channel'][0]['emission_lambda']) nwb.create_imaging_plane( name=meta_ip['name'], optical_channel=opt_ch, description=meta_ip['description'], device=device, excitation_lambda=meta_ip['excitation_lambda'], imaging_rate=fs, indicator=meta_ip['indicator'], location=meta_ip['location'], ) # Raw optical data if add_raw: print('Adding raw data...') for meta_tps in metadata['Ophys']['TwoPhotonSeries']: if meta_tps['name'][-1] == 'R': raw_data = file_raw['R'] else: raw_data = file_raw['Y'] def data_gen(data): xl, yl, zl, tl = data.shape chunk = 0 while chunk < tl: val = data[:, :, :, chunk] chunk += 1 print('adding data chunk: ', chunk) yield val xl, yl, zl, tl = raw_data.shape tps_data = DataChunkIterator(data=data_gen(data=raw_data), iter_axis=0, maxshape=(tl, xl, yl, zl)) # Change dimensions from (X,Y,Z,T) in mat file to (T,X,Y,Z) nwb standard #raw_data = np.moveaxis(raw_data, -1, 0) tps = TwoPhotonSeries( name=meta_tps['name'], imaging_plane=nwb.imaging_planes[meta_tps['imaging_plane']], data=tps_data, rate=file_info['info'].daq.scanRate) nwb.add_acquisition(tps) # Processed data if add_processed: print('Adding processed data...') ophys_module = ProcessingModule( name='Ophys', description='contains optical physiology processed data.', ) nwb.add_processing_module(ophys_module) # Create Image Segmentation compartment img_seg = ImageSegmentation( name=metadata['Ophys']['ImageSegmentation']['name']) ophys_module.add(img_seg) # Create plane segmentation and add ROIs meta_ps = metadata['Ophys']['ImageSegmentation'][ 'plane_segmentations'][0] ps = img_seg.create_plane_segmentation( name=meta_ps['name'], description=meta_ps['description'], imaging_plane=nwb.imaging_planes[meta_ps['imaging_plane']], ) # Add ROIs indices = file_sparse_matrix['indices'] indptr = file_sparse_matrix['indptr'] dims = np.squeeze(file_processed['dims']) for start, stop in zip(indptr, indptr[1:]): voxel_mask = make_voxel_mask(indices[start:stop], dims) ps.add_roi(voxel_mask=voxel_mask) # Visualize 3D voxel masks if plot_rois: plot_rois_function(plane_segmentation=ps, indptr=indptr) # DFF measures dff = DfOverF(name=metadata['Ophys']['DfOverF']['name']) ophys_module.add(dff) # create ROI regions n_cells = file_processed['dFF'].shape[0] roi_region = ps.create_roi_table_region(description='RoiTableRegion', region=list(range(n_cells))) # create ROI response series dff_data = file_processed['dFF'] tt = file_processed['time'].ravel() meta_rrs = metadata['Ophys']['DfOverF']['roi_response_series'][0] meta_rrs['data'] = dff_data.T meta_rrs['rois'] = roi_region meta_rrs['timestamps'] = tt dff.create_roi_response_series(**meta_rrs) # Creates GrayscaleVolume containers and add a reference image grayscale_volume = GrayscaleVolume( name=metadata['Ophys']['GrayscaleVolume']['name'], data=file_reference_image['im']) ophys_module.add(grayscale_volume) # Behavior data if add_behavior: print('Adding behavior data...') # Ball motion behavior_mod = nwb.create_processing_module( name='Behavior', description='holds processed behavior data', ) meta_ts = metadata['Behavior']['TimeSeries'][0] meta_ts['data'] = file_processed['ball'].ravel() tt = file_processed['time'].ravel() meta_ts['timestamps'] = tt behavior_ts = TimeSeries(**meta_ts) behavior_mod.add(behavior_ts) # Re-arranges spatial data of body-points positions tracking pos = file_processed['dlc'] n_points = 8 pos_reshaped = pos.reshape( (-1, n_points, 3)) # dims=(nSamples,n_points,3) # Creates a Position object and add one SpatialSeries for each body-point position position = Position() for i in range(n_points): position.create_spatial_series( name='SpatialSeries_' + str(i), data=pos_reshaped[:, i, :], timestamps=tt, reference_frame= 'Description defining what the zero-position is.', conversion=np.nan) behavior_mod.add(position) # Trial times trialFlag = file_processed['trialFlag'].ravel() trial_inds = np.hstack( (0, np.where(np.diff(trialFlag))[0], trialFlag.shape[0] - 1)) trial_times = tt[trial_inds] for start, stop in zip(trial_times, trial_times[1:]): nwb.add_trial(start_time=start, stop_time=stop) # Saves to NWB file with NWBHDF5IO(f_nwb, mode='w') as io: io.write(nwb) print('NWB file saved with size: ', os.stat(f_nwb).st_size / 1e6, ' mb')
def save(self, file_name, to32=True, order='F', imagej=False, bigtiff=True, excitation_lambda=488.0, compress=0, q_max=99.75, q_min=1, var_name_hdf5='mov', sess_desc='some_description', identifier='some identifier', imaging_plane_description='some imaging plane description', emission_lambda=520.0, indicator='OGB-1', location='brain', starting_time=0., experimenter='Dr Who', lab_name=None, institution=None, experiment_description='Experiment Description', session_id='Session ID'): """ Save the timeseries in single precision. Supported formats include TIFF, NPZ, AVI, MAT, HDF5/H5, MMAP, and NWB Args: file_name: str name of file. Possible formats are tif, avi, npz, mmap and hdf5 to32: Bool whether to transform to 32 bits order: 'F' or 'C' C or Fortran order var_name_hdf5: str Name of hdf5 file subdirectory q_max, q_min: float in [0, 100] percentile for maximum/minimum clipping value if saving as avi (If set to None, no automatic scaling to the dynamic range [0, 255] is performed) Raises: Exception 'Extension Unknown' """ name, extension = os.path.splitext(file_name)[:2] extension = extension.lower() logging.debug("Parsing extension " + str(extension)) if extension in ['.tif', '.tiff', '.btf']: with tifffile.TiffWriter(file_name, bigtiff=bigtiff, imagej=imagej) as tif: for i in range(self.shape[0]): if i % 200 == 0: logging.debug(str(i) + ' frames saved') curfr = self[i].copy() if to32 and not ('float32' in str(self.dtype)): curfr = curfr.astype(np.float32) tif.save(curfr, compress=compress) elif extension == '.npz': if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) np.savez(file_name, input_arr=input_arr, start_time=self.start_time, fr=self.fr, meta_data=self.meta_data, file_name=self.file_name) elif extension == '.avi': codec = None try: codec = cv2.FOURCC('I', 'Y', 'U', 'V') except AttributeError: codec = cv2.VideoWriter_fourcc(*'IYUV') if q_max is None or q_min is None: data = self.astype(np.uint8) else: if q_max < 100: maxmov = np.nanpercentile(self[::max(1, len(self) // 100)], q_max) else: maxmov = np.nanmax(self) if q_min > 0: minmov = np.nanpercentile(self[::max(1, len(self) // 100)], q_min) else: minmov = np.nanmin(self) data = 255 * (self - minmov) / (maxmov - minmov) np.clip(data, 0, 255, data) data = data.astype(np.uint8) y, x = data[0].shape vw = cv2.VideoWriter(file_name, codec, self.fr, (x, y), isColor=True) for d in data: vw.write(cv2.cvtColor(d, cv2.COLOR_GRAY2BGR)) vw.release() elif extension == '.mat': if self.file_name[0] is not None: f_name = self.file_name else: f_name = '' if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) if self.meta_data[0] is None: savemat( file_name, { 'input_arr': np.rollaxis(input_arr, axis=0, start=3), 'start_time': self.start_time, 'fr': self.fr, 'meta_data': [], 'file_name': f_name }) else: savemat( file_name, { 'input_arr': np.rollaxis(input_arr, axis=0, start=3), 'start_time': self.start_time, 'fr': self.fr, 'meta_data': self.meta_data, 'file_name': f_name }) elif extension in ('.hdf5', '.h5'): with h5py.File(file_name, "w") as f: if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) dset = f.create_dataset(var_name_hdf5, data=input_arr) dset.attrs["fr"] = self.fr dset.attrs["start_time"] = self.start_time try: dset.attrs["file_name"] = [ a.encode('utf8') for a in self.file_name ] except: logging.warning('No file saved') if self.meta_data[0] is not None: logging.debug("Metadata for saved file: " + str(self.meta_data)) dset.attrs["meta_data"] = cpk.dumps(self.meta_data) elif extension == '.mmap': base_name = name T = self.shape[0] dims = self.shape[1:] if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) input_arr = np.transpose(input_arr, list(range(1, len(dims) + 1)) + [0]) input_arr = np.reshape(input_arr, (np.prod(dims), T), order='F') fname_tot = memmap_frames_filename(base_name, dims, T, order) fname_tot = os.path.join(os.path.split(file_name)[0], fname_tot) big_mov = np.memmap(fname_tot, mode='w+', dtype=np.float32, shape=(np.uint64(np.prod(dims)), np.uint64(T)), order=order) big_mov[:] = np.asarray(input_arr, dtype=np.float32) big_mov.flush() del big_mov, input_arr return fname_tot elif extension == '.nwb': if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) # Create NWB file nwbfile = NWBFile(sess_desc, identifier, datetime.now(tzlocal()), experimenter=experimenter, lab=lab_name, institution=institution, experiment_description=experiment_description, session_id=session_id) # Get the device device = Device('imaging_device') nwbfile.add_device(device) # OpticalChannel optical_channel = OpticalChannel('OpticalChannel', 'main optical channel', emission_lambda=emission_lambda) imaging_plane = nwbfile.create_imaging_plane( name='ImagingPlane', optical_channel=optical_channel, description=imaging_plane_description, device=device, excitation_lambda=excitation_lambda, imaging_rate=self.fr, indicator=indicator, location=location) # Images image_series = TwoPhotonSeries(name=var_name_hdf5, dimension=self.shape[1:], data=input_arr, imaging_plane=imaging_plane, starting_frame=[0], starting_time=starting_time, rate=self.fr) nwbfile.add_acquisition(image_series) with NWBHDF5IO(file_name, 'w') as io: io.write(nwbfile) return file_name else: logging.error("Extension " + str(extension) + " unknown") raise Exception('Extension Unknown')
def add_ophys_processing_from_suite2p(save_folder, nwbfile, xml, device=None, optical_channel=None, imaging_plane=None, image_series=None): """ adapted from suite2p/suite2p/io/nwb.py "save_nwb" function """ plane_folders = natsorted([ f.path for f in os.scandir(save_folder) if f.is_dir() and f.name[:5]=='plane']) OPS = [np.load(os.path.join(f, 'ops.npy'), allow_pickle=True).item() for f in plane_folders] if len(OPS)>1: multiplane, nplanes = True, len(plane_folders) pData_folder = os.path.join(save_folder, 'combined') # processed data folder -> using the "combined output from suite2p" else: multiplane, nplanes = False, 1 pData_folder = os.path.join(save_folder, 'plane0') # processed data folder # find time sampling per plane functional_chan = ('Ch1' if len(xml['Ch1']['relativeTime'])>1 else 'Ch2') # functional channel is one of the two !! CaImaging_timestamps = xml[functional_chan]['relativeTime']+float(xml['settings']['framePeriod'])/2. ops = np.load(os.path.join(pData_folder, 'ops.npy'), allow_pickle=True).item() if device is None: device = nwbfile.create_device( name='Microscope', description='My two-photon microscope', manufacturer='The best microscope manufacturer') if optical_channel is None: optical_channel = OpticalChannel( name='OpticalChannel', description='an optical channel', emission_lambda=500.) if imaging_plane is None: imaging_plane = nwbfile.create_imaging_plane( name='ImagingPlane', optical_channel=optical_channel, imaging_rate=ops['fs'], description='standard', device=device, excitation_lambda=600., indicator='GCaMP', location='V1', grid_spacing=([2,2,30] if multiplane else [2,2]), grid_spacing_unit='microns') if image_series is None: # link to external data image_series = TwoPhotonSeries( name='TwoPhotonSeries', dimension=[ops['Ly'], ops['Lx']], external_file=(ops['filelist'] if 'filelist' in ops else ['']), imaging_plane=imaging_plane, starting_frame=[0], format='external', starting_time=0.0, rate=ops['fs'] * ops['nplanes'] ) nwbfile.add_acquisition(image_series) # otherwise, were added # processing img_seg = ImageSegmentation() ps = img_seg.create_plane_segmentation( name='PlaneSegmentation', description='suite2p output', imaging_plane=imaging_plane, reference_images=image_series ) ophys_module = nwbfile.create_processing_module( name='ophys', description='optical physiology processed data\n TSeries-folder=%s' % save_folder) ophys_module.add(img_seg) file_strs = ['F.npy', 'Fneu.npy', 'spks.npy'] traces = [] iscell = np.load(os.path.join(pData_folder, 'iscell.npy')).astype(bool) if ops['nchannels']>1: if os.path.isfile(os.path.join(pData_folder, 'redcell_manual.npy')): redcell = np.load(os.path.join(pData_folder, 'redcell_manual.npy'))[iscell[:,0], :] else: print('\n'+30*'--') print(' /!\ no file found for the manual labelling of red cells (generate it with the red-cell labelling GUI) /!\ ') print(' /!\ taking the raw suit2p output with the classifier settings /!\ ') print('\n'+30*'--') redcell = np.load(os.path.join(pData_folder, 'redcell.npy'))[iscell[:,0], :] for fstr in file_strs: traces.append(np.load(os.path.join(pData_folder, fstr))[iscell[:,0], :]) stat = np.load(os.path.join(pData_folder, 'stat.npy'), allow_pickle=True) ncells = np.sum(iscell[:,0]) plane_ID = np.zeros(ncells) for n in np.arange(ncells): pixel_mask = np.array([stat[iscell[:,0]][n]['ypix'], stat[iscell[:,0]][n]['xpix'], stat[iscell[:,0]][n]['lam']]) ps.add_roi(pixel_mask=pixel_mask.T) if 'iplane' in stat[0]: plane_ID[n] = stat[iscell[:,0]][n]['iplane'] if ops['nchannels']>1: ps.add_column('redcell', 'two columns - redcell & probcell', redcell) ps.add_column('plane', 'one column - plane ID', plane_ID) rt_region = ps.create_roi_table_region( region=list(np.arange(0, ncells)), description='all ROIs') # FLUORESCENCE (all are required) file_strs = ['F.npy', 'Fneu.npy', 'spks.npy'] name_strs = ['Fluorescence', 'Neuropil', 'Deconvolved'] for i, (fstr,nstr) in enumerate(zip(file_strs, name_strs)): roi_resp_series = RoiResponseSeries( name=nstr, data=traces[i], rois=rt_region, unit='lumens', timestamps=CaImaging_timestamps[::nplanes]) # ideally should be shifted for each ROI depending on the plane... fl = Fluorescence(roi_response_series=roi_resp_series, name=nstr) ophys_module.add(fl) # BACKGROUNDS # (meanImg, Vcorr and max_proj are REQUIRED) bg_strs = ['meanImg', 'meanImgE', 'Vcorr', 'max_proj', 'meanImg_chan2'] nplanes = ops['nplanes'] for iplane in range(nplanes): images = Images('Backgrounds_%d'%iplane) for bstr in bg_strs: if bstr in ops: if bstr=='Vcorr' or bstr=='max_proj': img = np.zeros((ops['Ly'], ops['Lx']), np.float32) img[ops['yrange'][0]:ops['yrange'][-1], ops['xrange'][0]:ops['xrange'][-1]] = ops[bstr] else: img = ops[bstr] images.add_image(GrayscaleImage(name=bstr, data=img)) ophys_module.add(images)