def test_init(self):
oc = OpticalChannel('test_name', 'test_source', 'description',
'emission_lambda')
self.assertEqual(oc.description, 'description')
self.assertEqual(oc.emission_lambda, 'emission_lambda')
ip = ImagingPlane('test_imaging_plane', 'test source', oc,
'description', 'device', 'excitation_lambda',
'imaging_rate', 'indicator', 'location',
(1, 2, 1, 2, 3), 4.0, 'unit', 'reference_frame')
self.assertEqual(ip.optical_channel[0], oc)
self.assertEqual(ip.device, 'device')
self.assertEqual(ip.excitation_lambda, 'excitation_lambda')
self.assertEqual(ip.imaging_rate, 'imaging_rate')
self.assertEqual(ip.indicator, 'indicator')
self.assertEqual(ip.location, 'location')
self.assertEqual(ip.manifold, (1, 2, 1, 2, 3))
self.assertEqual(ip.conversion, 4.0)
self.assertEqual(ip.unit, 'unit')
self.assertEqual(ip.reference_frame, 'reference_frame')
tPS = TwoPhotonSeries('test_tPS',
'a hypothetical source',
data=list(),
unit='unit',
field_of_view=list(),
imaging_plane=ip,
pmt_gain=1.0,
scan_line_rate=2.0,
external_file=['external_file'],
starting_frame=[1, 2, 3],
format='tiff',
timestamps=list())
self.assertEqual(tPS.name, 'test_tPS')
self.assertEqual(tPS.source, 'a hypothetical source')
self.assertEqual(tPS.unit, 'unit')
self.assertEqual(tPS.field_of_view, list())
self.assertEqual(tPS.imaging_plane, ip)
self.assertEqual(tPS.pmt_gain, 1.0)
self.assertEqual(tPS.scan_line_rate, 2.0)
self.assertEqual(tPS.external_file, ['external_file'])
self.assertEqual(tPS.starting_frame, [1, 2, 3])
self.assertEqual(tPS.format, 'tiff')
self.assertEqual(tPS.dimension, [np.nan])
def add_two_photon_series(imaging, nwbfile, metadata, buffer_size=10):
"""
Auxiliary static method for nwbextractor.
Adds two photon series from imaging object as TwoPhotonSeries to nwbfile object.
"""
metadata = dict_recursive_update(get_default_nwb_metadata(), metadata)
metadata = update_dict(metadata,
NwbImagingExtractor.get_nwb_metadata(imaging))
# Tests if ElectricalSeries already exists in acquisition
nwb_es_names = [ac for ac in nwbfile.acquisition]
opts = metadata["Ophys"]["TwoPhotonSeries"][0]
if opts["name"] not in nwb_es_names:
# retrieve device
device = nwbfile.devices[list(nwbfile.devices.keys())[0]]
metadata["Ophys"]["ImagingPlane"][0]["optical_channel"] = [
OpticalChannel(**i) for i in metadata["Ophys"]["ImagingPlane"]
[0]["optical_channel"]
]
metadata["Ophys"]["ImagingPlane"][0] = update_dict(
metadata["Ophys"]["ImagingPlane"][0], {"device": device})
imaging_plane = nwbfile.create_imaging_plane(
**metadata["Ophys"]["ImagingPlane"][0])
def data_generator(imaging):
for i in range(imaging.get_num_frames()):
yield imaging.get_frames(frame_idxs=[i]).T
data = H5DataIO(
DataChunkIterator(data_generator(imaging),
buffer_size=buffer_size),
compression=True,
)
# using internal data. this data will be stored inside the NWB file
two_p_series_kwargs = update_dict(
metadata["Ophys"]["TwoPhotonSeries"][0],
dict(data=data, imaging_plane=imaging_plane),
)
ophys_ts = TwoPhotonSeries(**two_p_series_kwargs)
nwbfile.add_acquisition(ophys_ts)
return nwbfile
def addContainer(self, file):
dev1 = file.create_device('dev1', 'dev1 description')
oc = OpticalChannel('optchan1', 'a fake OpticalChannel', 3.14)
ip = file.create_imaging_plane('imgpln1', oc, 'a fake ImagingPlane',
dev1, 6.28, 2.718, 'GFP',
'somewhere in the brain')
data = np.ones((3, 3, 3))
timestamps = list(range(10))
fov = [2.0, 2.0, 5.0]
tps = TwoPhotonSeries('test_2ps',
ip,
data,
'image_unit',
'raw',
fov,
1.7,
3.4,
timestamps=timestamps,
dimension=[200, 200])
file.add_acquisition(tps)
return tps
def test_init(self):
oc = OpticalChannel('test_name', 'description', 500.)
self.assertEqual(oc.description, 'description')
self.assertEqual(oc.emission_lambda, 500.)
device = Device(name='device_name')
ip = ImagingPlane('test_imaging_plane', oc, 'description', device,
600., 300., 'indicator', 'location', (50, 100, 3),
4.0, 'unit', 'reference_frame')
self.assertEqual(ip.optical_channel[0], oc)
self.assertEqual(ip.device, device)
self.assertEqual(ip.excitation_lambda, 600.)
self.assertEqual(ip.imaging_rate, 300.)
self.assertEqual(ip.indicator, 'indicator')
self.assertEqual(ip.location, 'location')
self.assertEqual(ip.manifold, (50, 100, 3))
self.assertEqual(ip.conversion, 4.0)
self.assertEqual(ip.unit, 'unit')
self.assertEqual(ip.reference_frame, 'reference_frame')
tPS = TwoPhotonSeries('test_tPS',
unit='unit',
field_of_view=[2., 3.],
imaging_plane=ip,
pmt_gain=1.0,
scan_line_rate=2.0,
external_file=['external_file'],
starting_frame=[1, 2, 3],
format='tiff',
timestamps=list())
self.assertEqual(tPS.name, 'test_tPS')
self.assertEqual(tPS.unit, 'unit')
self.assertEqual(tPS.field_of_view, [2., 3.])
self.assertEqual(tPS.imaging_plane, ip)
self.assertEqual(tPS.pmt_gain, 1.0)
self.assertEqual(tPS.scan_line_rate, 2.0)
self.assertEqual(tPS.external_file, ['external_file'])
self.assertEqual(tPS.starting_frame, [1, 2, 3])
self.assertEqual(tPS.format, 'tiff')
self.assertIsNone(tPS.dimension)
def test_init(self):
ip = create_imaging_plane()
tPS = TwoPhotonSeries(name='test_tPS',
unit='unit',
field_of_view=[2., 3.],
imaging_plane=ip,
pmt_gain=1.0,
scan_line_rate=2.0,
external_file=['external_file'],
starting_frame=[1, 2, 3],
format='tiff',
timestamps=list())
self.assertEqual(tPS.name, 'test_tPS')
self.assertEqual(tPS.unit, 'unit')
self.assertEqual(tPS.field_of_view, [2., 3.])
self.assertEqual(tPS.imaging_plane, ip)
self.assertEqual(tPS.pmt_gain, 1.0)
self.assertEqual(tPS.scan_line_rate, 2.0)
self.assertEqual(tPS.external_file, ['external_file'])
self.assertEqual(tPS.starting_frame, [1, 2, 3])
self.assertEqual(tPS.format, 'tiff')
self.assertIsNone(tPS.dimension)
def addContainer(self, file):
oc = OpticalChannel('optchan1', 'unit test TestImagingPlaneIO',
'a fake OpticalChannel', '3.14')
ip = ImagingPlane('imgpln1', 'unit test TestImagingPlaneIO', oc,
'a fake ImagingPlane', 'imaging_device_1', '6.28',
'2.718', 'GFP', 'somewhere in the brain')
file.set_imaging_plane(ip)
data = list(zip(range(10), range(10, 20)))
timestamps = list(range(10))
fov = [2.0, 2.0, 5.0]
tps = TwoPhotonSeries('test_2ps',
'unit test TestTwoPhotonSeries',
data,
ip,
'image_unit',
'raw',
fov,
1.7,
3.4,
timestamps=timestamps,
dimension=[2])
file.add_acquisition(tps)
def setUpClass(self):
device = Device("imaging_device_1")
optical_channel = OpticalChannel("my_optchan", "description", 500.0)
self.imaging_plane = ImagingPlane(
name="imgpln1",
optical_channel=optical_channel,
description="a fake ImagingPlane",
device=device,
excitation_lambda=600.0,
imaging_rate=300.0,
indicator="GFP",
location="somewhere in the brain",
reference_frame="unknown",
origin_coords=[10, 20],
origin_coords_unit="millimeters",
grid_spacing=[0.001, 0.001],
grid_spacing_unit="millimeters",
)
self.image_series = TwoPhotonSeries(
name="test_image_series",
data=np.random.randn(100, 5, 5),
imaging_plane=self.imaging_plane,
starting_frame=[0],
rate=1.0,
unit="n.a",
)
self.img_seg = ImageSegmentation()
self.ps2 = self.img_seg.create_plane_segmentation(
"output from segmenting my favorite imaging plane",
self.imaging_plane,
"2d_plane_seg",
self.image_series,
)
self.ps2.add_column("type", "desc")
self.ps2.add_column("type2", "desc")
w, h = 3, 3
img_mask1 = np.zeros((w, h))
img_mask1[0, 0] = 1.1
img_mask1[1, 1] = 1.2
img_mask1[2, 2] = 1.3
self.ps2.add_roi(image_mask=img_mask1, type=1, type2=0)
img_mask2 = np.zeros((w, h))
img_mask2[0, 0] = 2.1
img_mask2[1, 1] = 2.2
self.ps2.add_roi(image_mask=img_mask2, type=1, type2=1)
img_mask2 = np.zeros((w, h))
img_mask2[0, 0] = 9.1
img_mask2[1, 1] = 10.2
self.ps2.add_roi(image_mask=img_mask2, type=2, type2=0)
img_mask2 = np.zeros((w, h))
img_mask2[0, 0] = 3.5
img_mask2[1, 1] = 5.6
self.ps2.add_roi(image_mask=img_mask2, type=2, type2=1)
fl = Fluorescence()
rt_region = self.ps2.create_roi_table_region("the first of two ROIs",
region=[0, 1, 2, 3])
rois_shape = 5
data = np.arange(10 * rois_shape).reshape([10, -1], order='F')
timestamps = np.array(
[0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9])
rrs = fl.create_roi_response_series(name="my_rrs",
data=data,
rois=rt_region,
unit="lumens",
timestamps=timestamps)
self.df_over_f = DfOverF(rrs)
device='imaging_device_1',
excitation_lambda='unknown',
imaging_rate='unknown',
indicator='GFP',
location='unknown',
manifold=np.array([]),
conversion=1.0,
unit='um',
reference_frame='A frame to refer to')
for channel_name, imaging_data in zip(channel_names, all_imaging_data):
image_series = TwoPhotonSeries(name='image',
source='Ca2+ imaging example',
dimension=[2],
data=imaging_data,
imaging_plane=imaging_plane,
starting_frame=[0],
timestamps=[1, 2, 3],
scan_line_rate=np.nan,
pmt_gain=np.nan)
nwbfile.add_acquisition(image_series)
out_fname = 'sebi_data.nwb'
print('writing NWB file...', end='', flush=True)
with NWBHDF5IO(out_fname, mode='w') as io:
io.write(nwbfile, cache_spec=False)
print('done.')
print('testing read...', end='', flush=True)
# test read
with NWBHDF5IO(out_fname, mode='r') as io:
device=device,
excitation_lambda=np.nan,
imaging_rate=np.nan,
indicator='GFP',
location='unknown',
manifold=np.array([]),
conversion=1.0,
unit='um',
reference_frame='A frame to refer to')
for channel_name, imaging_data in zip(channel_names, all_imaging_data):
image_series = TwoPhotonSeries(name='TwoPhotonSeries' +
channel_name.decode(),
dimension=[2],
data=H5DataIO(imaging_data,
compression='gzip'),
imaging_plane=imaging_plane,
starting_frame=[0],
timestamps=[1, 2, 3],
scan_line_rate=np.nan,
pmt_gain=np.nan)
nwbfile.add_acquisition(image_series)
if SHORTEN:
out_fname = fpath + '_stub.nwb'
else:
out_fname = fpath + '.nwb'
print('writing NWB file...', end='', flush=True)
with NWBHDF5IO(out_fname, mode='w') as io:
io.write(nwbfile)
print('done.')
def save_nwb(ops1):
if NWB and not ops1[0]['mesoscan']:
if len(ops1) > 1:
multiplane = True
else:
multiplane = False
ops = ops1[0]
### INITIALIZE NWB FILE
nwbfile = NWBFile(
session_description='suite2p_proc',
identifier=ops['data_path'][0],
session_start_time=(ops['date_proc'] if 'date_proc' in ops else
datetime.datetime.now()))
print(nwbfile)
device = nwbfile.create_device(
name='Microscope',
description='My two-photon microscope',
manufacturer='The best microscope manufacturer')
optical_channel = OpticalChannel(name='OpticalChannel',
description='an optical channel',
emission_lambda=500.)
imaging_plane = nwbfile.create_imaging_plane(
name='ImagingPlane',
optical_channel=optical_channel,
imaging_rate=ops['fs'],
description='standard',
device=device,
excitation_lambda=600.,
indicator='GCaMP',
location='V1',
grid_spacing=([2, 2, 30] if multiplane else [2, 2]),
grid_spacing_unit='microns')
# link to external data
image_series = TwoPhotonSeries(
name='TwoPhotonSeries',
dimension=[ops['Ly'], ops['Lx']],
external_file=(ops['filelist'] if 'filelist' in ops else ['']),
imaging_plane=imaging_plane,
starting_frame=[0],
format='external',
starting_time=0.0,
rate=ops['fs'] * ops['nplanes'])
nwbfile.add_acquisition(image_series)
# processing
img_seg = ImageSegmentation()
ps = img_seg.create_plane_segmentation(name='PlaneSegmentation',
description='suite2p output',
imaging_plane=imaging_plane,
reference_images=image_series)
ophys_module = nwbfile.create_processing_module(
name='ophys', description='optical physiology processed data')
ophys_module.add(img_seg)
file_strs = ['F.npy', 'Fneu.npy', 'spks.npy']
traces = []
ncells_all = 0
for iplane, ops in enumerate(ops1):
if iplane == 0:
iscell = np.load(os.path.join(ops['save_path'], 'iscell.npy'))
for fstr in file_strs:
traces.append(np.load(os.path.join(ops['save_path'],
fstr)))
else:
iscell = np.append(iscell,
np.load(
os.path.join(ops['save_path'],
'iscell.npy')),
axis=0)
for i, fstr in enumerate(file_strs):
traces[i] = np.append(
traces[i],
np.load(os.path.join(ops['save_path'], fstr)),
axis=0)
stat = np.load(os.path.join(ops['save_path'], 'stat.npy'),
allow_pickle=True)
ncells = len(stat)
for n in range(ncells):
if multiplane:
pixel_mask = np.array([
stat[n]['ypix'], stat[n]['xpix'],
iplane * np.ones(stat[n]['npix']), stat[n]['lam']
])
ps.add_roi(voxel_mask=pixel_mask.T)
else:
pixel_mask = np.array(
[stat[n]['ypix'], stat[n]['xpix'], stat[n]['lam']])
ps.add_roi(pixel_mask=pixel_mask.T)
ncells_all += ncells
ps.add_column('iscell', 'two columns - iscell & probcell', iscell)
rt_region = ps.create_roi_table_region(region=list(
np.arange(0, ncells_all)),
description='all ROIs')
# FLUORESCENCE (all are required)
file_strs = ['F.npy', 'Fneu.npy', 'spks.npy']
name_strs = ['Fluorescence', 'Neuropil', 'Deconvolved']
for i, (fstr, nstr) in enumerate(zip(file_strs, name_strs)):
roi_resp_series = RoiResponseSeries(name=nstr,
data=traces[i],
rois=rt_region,
unit='lumens',
rate=ops['fs'])
fl = Fluorescence(roi_response_series=roi_resp_series, name=nstr)
ophys_module.add(fl)
# BACKGROUNDS
# (meanImg, Vcorr and max_proj are REQUIRED)
bg_strs = ['meanImg', 'Vcorr', 'max_proj', 'meanImg_chan2']
nplanes = ops['nplanes']
for iplane in range(nplanes):
images = Images('Backgrounds_%d' % iplane)
for bstr in bg_strs:
if bstr in ops:
if bstr == 'Vcorr' or bstr == 'max_proj':
img = np.zeros((ops['Ly'], ops['Lx']), np.float32)
img[ops['yrange'][0]:ops['yrange'][-1],
ops['xrange'][0]:ops['xrange'][-1]] = ops[bstr]
else:
img = ops[bstr]
images.add_image(GrayscaleImage(name=bstr, data=img))
ophys_module.add(images)
with NWBHDF5IO(os.path.join(ops['save_path0'], 'suite2p', 'ophys.nwb'),
'w') as fio:
fio.write(nwbfile)
else:
print('pip install pynwb OR don"t use mesoscope recording')
def setUp(self):
start_time = datetime(2017, 4, 3, 11, 0, 0)
create_date = datetime(2017, 4, 15, 12, 0, 0)
# create your NWBFile object
nwbfile = NWBFile('PyNWB Sample File', 'A simple NWB file', 'NWB_test', start_time,
file_create_date=create_date)
# create acquisition metadata
optical_channel = OpticalChannel('test_optical_channel', 'optical channel source',
'optical channel description', 3.14)
imaging_plane = nwbfile.create_imaging_plane('test_imaging_plane',
'ophys integration tests',
optical_channel,
'imaging plane description',
'imaging_device_1',
6.28, '2.718', 'GFP', 'somewhere in the brain',
(1, 2, 1, 2, 3), 4.0, 'manifold unit', 'A frame to refer to')
# create acquisition data
image_series = TwoPhotonSeries(name='test_iS', source='a hypothetical source', dimension=[2],
external_file=['images.tiff'], imaging_plane=imaging_plane,
starting_frame=[1, 2, 3], format='tiff', timestamps=list())
nwbfile.add_acquisition(image_series)
mod = nwbfile.create_processing_module('img_seg_example', 'ophys demo', 'an example of writing Ca2+ imaging data')
img_seg = ImageSegmentation('a toy image segmentation container')
mod.add_data_interface(img_seg)
ps = img_seg.create_plane_segmentation('integration test PlaneSegmentation', 'plane segmentation description',
imaging_plane, 'test_plane_seg_name', image_series)
# add two ROIs
# - first argument is the pixel mask i.e. a list of pixels and their weights
# - second argument is the image mask
w, h = 3, 3
pix_mask1 = [(0, 0, 1.1), (1, 1, 1.2), (2, 2, 1.3)]
img_mask1 = [[0.0 for x in range(w)] for y in range(h)]
img_mask1[0][0] = 1.1
img_mask1[1][1] = 1.2
img_mask1[2][2] = 1.3
ps.add_roi('1234', pix_mask1, img_mask1)
pix_mask2 = [(0, 0, 2.1), (1, 1, 2.2)]
img_mask2 = [[0.0 for x in range(w)] for y in range(h)]
img_mask2[0][0] = 2.1
img_mask2[1][1] = 2.2
ps.add_roi('5678', pix_mask2, img_mask2)
# add a Fluorescence container
fl = Fluorescence('a toy fluorescence container')
mod.add_data_interface(fl)
# get an ROI table region i.e. a subset of ROIs to create a RoiResponseSeries from
rt_region = ps.create_roi_table_region('the first of two ROIs', region=[0])
# make some fake timeseries data
data = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
timestamps = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
rrs = fl.create_roi_response_series('test_roi_response_series', 'RoiResponseSeries integration test',
data, 'lumens', rt_region, timestamps=timestamps)
# write data
nwb_path = './test_data/nwb_test_file.nwb'
with NWBHDF5IO(nwb_path, 'w') as io:
io.write(nwbfile)
def add_two_photon_series(imaging, nwbfile, metadata, num_chunks=10):
"""
Auxiliary static method for nwbextractor.
Adds two photon series from imaging object as TwoPhotonSeries to nwbfile object.
"""
if 'Ophys' not in metadata:
metadata['Ophys'] = {}
if 'TwoPthotonSeries' not in metadata['Ophys']:
metadata['Ophys']['TwoPhotonSeries'] = [{
'name':
'TwoPhotonSeries',
'description':
'optical_series_description'
}]
# Tests if ElectricalSeries already exists in acquisition
nwb_es_names = [ac for ac in nwbfile.acquisition]
opts = metadata['Ophys']['TwoPhotonSeries'][0]
if opts['name'] not in nwb_es_names:
# retrieve device
device = nwbfile.devices[list(nwbfile.devices.keys())[0]]
# create optical channel
if 'OpticalChannel' not in metadata['Ophys']:
metadata['Ophys']['OpticalChannel'] = [{
'name':
'OpticalChannel',
'description':
'no description',
'emission_lambda':
np.nan
}]
optical_channel = OpticalChannel(
**metadata['Ophys']['OpticalChannel'][0])
# sampling rate
rate = float(imaging.get_sampling_frequency())
if 'ImagingPlane' not in metadata['Ophys']:
metadata['Ophys']['ImagingPlane'] = [{
'name': 'ImagingPlane',
'description': 'no description',
'excitation_lambda': np.nan,
'indicator': 'unknown',
'location': 'unknown'
}]
imaging_meta = {
'optical_channel': optical_channel,
'imaging_rate': rate,
'device': device
}
metadata['Ophys']['ImagingPlane'][0] = update_dict(
metadata['Ophys']['ImagingPlane'][0], imaging_meta)
imaging_plane = nwbfile.create_imaging_plane(
**metadata['Ophys']['ImagingPlane'][0])
def data_generator(imaging, num_chunks):
num_frames = imaging.get_num_frames()
# chunk size is not None
chunk_size = num_frames // num_chunks
if num_frames % chunk_size > 0:
num_chunks += 1
for i in range(num_chunks):
video = imaging.get_video(start_frame=i * chunk_size,
end_frame=min(
(i + 1) * chunk_size,
num_frames))
data = np.squeeze(video)
yield data
data = H5DataIO(DataChunkIterator(
data_generator(imaging, num_chunks)),
compression=True)
acquisition_name = opts['name']
# using internal data. this data will be stored inside the NWB file
ophys_ts = TwoPhotonSeries(
name=acquisition_name,
data=data,
imaging_plane=imaging_plane,
rate=rate,
unit='normalized amplitude',
comments='Generated from RoiInterface::NwbImagingExtractor',
description='no description')
nwbfile.add_acquisition(ophys_ts)
return nwbfile
def add_imaging(nwbfile,
expt,
z_spacing=25.,
device_name='2P Microscope',
location='CA1',
indicator='GCaMP6f',
excitation_lambda=920.,
data_root=None,
stub=False):
color_dict = {'Ch1': 'Red', 'Ch2': 'Green'}
# Emissions for mCherry and GCaMP
# TODO make this more flexible
emission = {'Ch1': 640., 'Ch2': 530.}
ch_names = ['Ch1', 'Ch2']
optical_channels = []
for ch_name in ch_names:
optical_channel = OpticalChannel(name=ch_name,
description=color_dict[ch_name],
emission_lambda=emission[ch_name])
optical_channels.append(optical_channel)
h5_path = glob(os.path.join(data_root, '*.h5'))[0]
pv_xml = os.path.join(data_root, os.path.basename(data_root) + '.xml')
pv_version = get_prairieview_version(pv_xml)
[y_um, x_um] = get_element_size_um(pv_xml, pv_version)[-2:]
elem_size_um = [z_spacing, y_um, x_um]
# TODO allow for flexibility in setting device, excitation, indicator, location
# TODO nwb-schema issue #151 needs to be resolved so we can actually use imaging data size
device = nwbfile.create_device(device_name)
imaging_plane = nwbfile.create_imaging_plane(
name='Imaging Data',
optical_channel=optical_channels,
description='imaging data for both channels',
device=device,
excitation_lambda=excitation_lambda,
imaging_rate=1 / expt.frame_period(),
indicator=indicator,
location=location,
conversion=1.0, # Should actually be elem_size_um
manifold=np.ones((2, 2, 2, 3)),
reference_frame='reference_frame',
unit='um')
f = h5py.File(h5_path, 'r')
imaging_data = f['imaging']
channel_names = f['imaging'].attrs['channel_names']
for c, channel_name in enumerate(channel_names):
if not stub:
data_in = H5DataIO(DataChunkIterator(tqdm(
(np.swapaxes(data[..., c], 0, 2) for data in imaging_data),
total=imaging_data.shape[0]),
buffer_size=5000),
compression='gzip')
else:
data_in = np.ones((10, 10, 10)) # use for dev testing for speed
# TODO parse env file to add power and pmt gain?
image_series = TwoPhotonSeries(name='2p_Series_' + channel_name,
dimension=expt.frame_shape()[:-1],
data=data_in,
imaging_plane=imaging_plane,
rate=1 / expt.frame_period(),
starting_time=0.,
description=channel_name)
nwbfile.add_acquisition(image_series)
imaging_plane = nwbfile.create_imaging_plane(name='Culture',
optical_channel=optical_channel_2,
imaging_rate=400.,
description='Voltage imaging',
device=device_2,
excitation_lambda=000.,
indicator='Quasar-2',
location='')
# grid_spacing=[.01, .01], ## how much µm is 1 pixel ?
# grid_spacing_unit='µm')
# using internal data. this data will be stored inside the NWB file as acquired data that is not-modifiable
# can add multiple series if wanted
image_series1 = TwoPhotonSeries(name='fluorescence data',
data=data,
imaging_plane=imaging_plane,
rate=400.0,
unit='pixel intensity')
nwbfile.add_acquisition(image_series1)
## Create a ProcessingModule to store the future processed data
ophys_module = nwbfile.create_processing_module(
name='ophys', description='optical physiology processed data')
## ImageSegmentation object and add the ImageSegmentation to the ophy module.
img_seg = ImageSegmentation(
) ## imagesegmentation can contain multiple planesegmentation
ophys_module.add(img_seg)
## Create PlaneSegmentation tables within the targetted imaging_plane
ps = img_seg.create_plane_segmentation(
####################
# Adding two-photon image data
# ----------------------------
#
# Now that you have your :py:class:`~pynwb.ophys.ImagingPlane`, you can create a
# :py:class:`~pynwb.ophys.TwoPhotonSeries` - the class representing two photon imaging data.
#
# From here you have two options. The first option is to supply the image data to PyNWB, using the `data` argument.
# The other option is the provide a path the images. These two options have trade-offs, so it is worth spending time
# considering how you want to store this data [#]_.
image_series = TwoPhotonSeries(name='test_iS', source='Ca2+ imaging example', dimension=[2],
external_file=['images.tiff'], imaging_plane=imaging_plane,
starting_frame=[0], format='tiff', timestamps=list())
nwbfile.add_acquisition(image_series)
####################
# Storing image segmentation output
# ---------------------------------
#
# Now that the raw data is stored, you can add the image segmentation results. This is done with the
# :py:class:`~pynwb.ophys.ImageSegmentation` data interface. This class has the ability to store segmentation
# from one or more imaging planes; hence the :py:class:`~pynwb.ophys.PlaneSegmentation` class.
mod = nwbfile.create_processing_module('my_ca_imaging_module', 'Ca2+ imaging example', 'example data module')
img_seg = ImageSegmentation('Ca2+ imaging example')
def add_ophys_processing_from_suite2p(save_folder,
nwbfile,
CaImaging_timestamps,
device=None,
optical_channel=None,
imaging_plane=None,
image_series=None):
"""
adapted from suite2p/suite2p/io/nwb.py "save_nwb" function
"""
plane_folders = natsorted([
f.path for f in os.scandir(save_folder)
if f.is_dir() and f.name[:5] == 'plane'
])
ops1 = [
np.load(os.path.join(f, 'ops.npy'), allow_pickle=True).item()
for f in plane_folders
]
if len(ops1) > 1:
multiplane = True
else:
multiplane = False
ops = ops1[0]
if device is None:
device = nwbfile.create_device(
name='Microscope',
description='My two-photon microscope',
manufacturer='The best microscope manufacturer')
if optical_channel is None:
optical_channel = OpticalChannel(name='OpticalChannel',
description='an optical channel',
emission_lambda=500.)
if imaging_plane is None:
imaging_plane = nwbfile.create_imaging_plane(
name='ImagingPlane',
optical_channel=optical_channel,
imaging_rate=ops['fs'],
description='standard',
device=device,
excitation_lambda=600.,
indicator='GCaMP',
location='V1',
grid_spacing=([2, 2, 30] if multiplane else [2, 2]),
grid_spacing_unit='microns')
if image_series is None:
# link to external data
image_series = TwoPhotonSeries(
name='TwoPhotonSeries',
dimension=[ops['Ly'], ops['Lx']],
external_file=(ops['filelist'] if 'filelist' in ops else ['']),
imaging_plane=imaging_plane,
starting_frame=[0],
format='external',
starting_time=0.0,
rate=ops['fs'] * ops['nplanes'])
nwbfile.add_acquisition(image_series) # otherwise, were added
# processing
img_seg = ImageSegmentation()
ps = img_seg.create_plane_segmentation(name='PlaneSegmentation',
description='suite2p output',
imaging_plane=imaging_plane,
reference_images=image_series)
ophys_module = nwbfile.create_processing_module(
name='ophys', description='optical physiology processed data')
ophys_module.add(img_seg)
file_strs = ['F.npy', 'Fneu.npy', 'spks.npy']
traces = []
ncells_all = 0
for iplane, ops in enumerate(ops1):
if iplane == 0:
iscell = np.load(
os.path.join(save_folder, 'plane%i' % iplane, 'iscell.npy'))
for fstr in file_strs:
traces.append(
np.load(os.path.join(save_folder, 'plane%i' % iplane,
fstr)))
else:
iscell = np.append(iscell,
np.load(
os.path.join(save_folder,
'plane%i' % iplane,
'iscell.npy')),
axis=0)
for i, fstr in enumerate(file_strs):
traces[i] = np.append(
traces[i],
np.load(os.path.join(save_folder, 'plane%i' % iplane,
fstr)),
axis=0)
stat = np.load(os.path.join(save_folder, 'plane%i' % iplane,
'stat.npy'),
allow_pickle=True)
ncells = len(stat)
for n in range(ncells):
if multiplane:
pixel_mask = np.array([
stat[n]['ypix'], stat[n]['xpix'],
iplane * np.ones(stat[n]['npix']), stat[n]['lam']
])
ps.add_roi(voxel_mask=pixel_mask.T)
else:
pixel_mask = np.array(
[stat[n]['ypix'], stat[n]['xpix'], stat[n]['lam']])
ps.add_roi(pixel_mask=pixel_mask.T)
ncells_all += ncells
ps.add_column('iscell', 'two columns - iscell & probcell', iscell)
rt_region = ps.create_roi_table_region(region=list(np.arange(
0, ncells_all)),
description='all ROIs')
# FLUORESCENCE (all are required)
file_strs = ['F.npy', 'Fneu.npy', 'spks.npy']
name_strs = ['Fluorescence', 'Neuropil', 'Deconvolved']
for i, (fstr, nstr) in enumerate(zip(file_strs, name_strs)):
roi_resp_series = RoiResponseSeries(
name=nstr,
data=traces[i],
rois=rt_region,
unit='lumens',
timestamps=CaImaging_timestamps
) # CRITICAL TO HAVE IT HERE FOR RE-ALIGNEMENT
fl = Fluorescence(roi_response_series=roi_resp_series, name=nstr)
ophys_module.add(fl)
# BACKGROUNDS
# (meanImg, Vcorr and max_proj are REQUIRED)
bg_strs = ['meanImg', 'meanImgE', 'Vcorr', 'max_proj', 'meanImg_chan2']
nplanes = ops['nplanes']
for iplane in range(nplanes):
images = Images('Backgrounds_%d' % iplane)
for bstr in bg_strs:
if bstr in ops:
if bstr == 'Vcorr' or bstr == 'max_proj':
img = np.zeros((ops['Ly'], ops['Lx']), np.float32)
img[ops['yrange'][0]:ops['yrange'][-1],
ops['xrange'][0]:ops['xrange'][-1]] = ops[bstr]
else:
img = ops[bstr]
images.add_image(GrayscaleImage(name=bstr, data=img))
ophys_module.add(images)
def convert(self, **kwargs):
"""Convert the data and add to the nwb_file
Args:
**kwargs: arbitrary arguments
"""
super().convert(**kwargs)
# ### setting parameters ####
formats_implemented = ["external", "tiff"]
if not kwargs.get("format"):
raise Exception(f"'format' argument should be pass to convert function in class {self.__class__.__name__}")
elif kwargs["format"] not in formats_implemented:
raise Exception(f"'format' argument should have one of these values {formats_implemented} "
f"for the convert function in class {self.__class__.__name__}")
movie_format = kwargs["format"]
if not kwargs.get("motion_corrected_file_name"):
raise Exception(f"'motion_corrected_file_name' attribute should be pass to convert "
f"function in class {self.__class__.__name__}")
motion_corrected_file_name = kwargs["motion_corrected_file_name"]
if "original_movie_file_name" in kwargs:
original_movie_file_name = kwargs["original_movie_file_name"]
else:
original_movie_file_name = None
if "xy_translation_file_name" in kwargs:
xy_translation_file_name = kwargs["xy_translation_file_name"]
else:
xy_translation_file_name = None
# Open YAML file with metadata if existing then dump all data in a dict
if ("yaml_file_name" in kwargs) and kwargs["yaml_file_name"] is not None:
with open(kwargs["yaml_file_name"], 'r') as stream:
yaml_data = yaml.safe_load(stream)
else:
raise Exception(f"'yaml_file_name' attribute should be pass to convert "
f"function in class {self.__class__.__name__}")
# a calcium imaging rate has to be given, either trought the yaml file, either as argument
# self.ci_sampling_rate can be obtained by the abf_converter
if "imaging_rate" in yaml_data:
self.ci_sampling_rate = yaml_data["imaging_rate"]
elif "ci_sampling_rate" in kwargs:
self.ci_sampling_rate = kwargs["ci_sampling_rate"]
else:
raise Exception(f"No 'imaging_rate' provided for movie {motion_corrected_file_name} in the yaml file "
f"{kwargs['yaml_file_name']} or throught argument 'ci_sampling_rate' to function convert() "
f"of the class {self.__class__.__name__}")
if "indicator" in yaml_data:
indicator = yaml_data["indicator"]
else:
raise Exception(f"No 'indicator' provided for movie {motion_corrected_file_name} in the yaml file "
f"{kwargs['yaml_file_name']}")
if "excitation_lambda" in yaml_data:
excitation_lambda = yaml_data["excitation_lambda"]
else:
raise Exception(f"No 'excitation_lambda' provided for movie {motion_corrected_file_name} in the yaml file "
f"{kwargs['yaml_file_name']}")
if "emission_lambda" in yaml_data:
emission_lambda = yaml_data["emission_lambda"]
else:
raise Exception(f"No 'emission_lambda' provided for movie {motion_corrected_file_name} in the yaml file "
f"{kwargs['yaml_file_name']}")
if "image_plane_location" in yaml_data:
image_plane_location = yaml_data["image_plane_location"]
else:
raise Exception(
f"No 'image_plane_location' provided for movie {motion_corrected_file_name} in the yaml file "
f"{kwargs['yaml_file_name']}")
try :
if 'ci_recording_on_pause' in self.nwb_file.intervals:
pause_intervals = self.nwb_file.intervals['ci_recording_on_pause']
pause_intervals_df = pause_intervals.to_dataframe()
start_times = pause_intervals_df.loc[:, "start_time"]
stop_times = pause_intervals_df.loc[:, "stop_time"]
try:
ci_frames_time_series = self.nwb_file.get_acquisition("ci_frames")
ci_frames = np.where(ci_frames_time_series.data)[0]
ci_frames_timestamps = ci_frames_time_series.timestamps[ci_frames]
for i, start_time in enumerate(start_times):
frame_index = np.searchsorted(a=ci_frames_timestamps, v=start_time)
n_frames_to_add = (stop_times[i] - start_time) * self.ci_sampling_rate
self.frames_to_add[frame_index] = int(n_frames_to_add)
except KeyError:
pass
except AttributeError:
pass
# ### end setting parameters ####
device = Device('2P_device')
self.nwb_file.add_device(device)
optical_channel = OpticalChannel('my_optchan', 'description', emission_lambda)
imaging_plane = self.nwb_file.create_imaging_plane(name='my_imgpln',
optical_channel=optical_channel,
description='a very interesting part of the brain',
device=device,
excitation_lambda=excitation_lambda,
imaging_rate=float(self.ci_sampling_rate),
indicator=indicator,
location=image_plane_location)
if movie_format != "external":
tiff_movie = self.load_tiff_movie_in_memory(motion_corrected_file_name)
dim_y, dim_x = tiff_movie.shape[1:]
n_frames = tiff_movie.shape[0]
motion_corrected_img_series = TwoPhotonSeries(name='motion_corrected_ci_movie', dimension=[dim_x, dim_y],
data=tiff_movie,
imaging_plane=imaging_plane,
starting_frame=[0], format=movie_format,
rate=self.ci_sampling_rate)
if original_movie_file_name is not None:
original_tiff_movie = self.load_tiff_movie_in_memory(original_movie_file_name)
dim_y, dim_x = original_tiff_movie.shape[1:]
original_img_series = TwoPhotonSeries(name='original_ci_movie', dimension=[dim_x, dim_y],
data=original_tiff_movie,
imaging_plane=imaging_plane,
starting_frame=[0], format=movie_format,
rate=float(self.ci_sampling_rate))
else:
im = PIL.Image.open(motion_corrected_file_name)
n_frames = len(list(ImageSequence.Iterator(im)))
dim_y, dim_x = np.array(im).shape
motion_corrected_img_series = TwoPhotonSeries(name='motion_corrected_ci_movie', dimension=[dim_x, dim_y],
external_file=[motion_corrected_file_name],
imaging_plane=imaging_plane,
starting_frame=[0], format=movie_format,
rate=float(self.ci_sampling_rate))
if original_movie_file_name is not None:
im = PIL.Image.open(original_movie_file_name)
dim_y, dim_x = np.array(im).shape
original_img_series = TwoPhotonSeries(name='original_ci_movie',
dimension=[dim_x, dim_y],
external_file=[original_movie_file_name],
imaging_plane=imaging_plane,
starting_frame=[0], format=movie_format,
rate=float(self.ci_sampling_rate))
self.nwb_file.add_acquisition(motion_corrected_img_series)
if original_movie_file_name is not None:
self.nwb_file.add_acquisition(original_img_series)
if xy_translation_file_name is not None:
if xy_translation_file_name.endswith(".mat"):
mvt_x_y = hdf5storage.loadmat(os.path.join(xy_translation_file_name))
x_shifts = mvt_x_y['xshifts'][0]
y_shifts = mvt_x_y['yshifts'][0]
elif xy_translation_file_name.endswith(".npy"):
ops = np.load(os.path.join(xy_translation_file_name))
ops = ops.item()
x_shifts = ops['xoff']
y_shifts = ops['yoff']
xy_translation = np.zeros((n_frames, 2), dtype="int16")
frame_index = 0
for frame in np.arange(len(x_shifts)):
xy_translation[frame_index, 0] = x_shifts[frame]
xy_translation[frame_index, 1] = y_shifts[frame]
# adding frames is necessary, in case the movie would be a concatenation of movie for exemple
if frame in self.frames_to_add:
frame_index += self.frames_to_add[frame]
xy_translation_time_series = TimeSeries(name="xy_translation", data=xy_translation)
corrected_image_stack = CorrectedImageStack(name="CorrectedImageStack",
corrected=motion_corrected_img_series,
original=original_img_series,
xy_translation=xy_translation_time_series)
self.nwb_file.add_acquisition(corrected_image_stack)
def setUp(self):
nwbfile = NWBFile(
'my first synthetic recording',
'EXAMPLE_ID',
datetime.now(tzlocal()),
experimenter='Dr. Bilbo Baggins',
lab='Bag End Laboratory',
institution='University of Middle Earth at the Shire',
experiment_description=('I went on an adventure with thirteen '
'dwarves to reclaim vast treasures.'),
session_id='LONELYMTN')
device = Device('imaging_device_1')
nwbfile.add_device(device)
optical_channel = OpticalChannel('my_optchan', 'description', 500.)
imaging_plane = nwbfile.create_imaging_plane(
'my_imgpln', optical_channel,
'a very interesting part of the brain',
device, 600., 300., 'GFP', 'my favorite brain location',
np.ones((5, 5, 3)), 4.0, 'manifold unit', 'A frame to refer to')
self.image_series = TwoPhotonSeries(name='test_iS',
dimension=[2],
data=np.random.rand(10, 5, 5, 3),
external_file=['images.tiff'],
imaging_plane=imaging_plane,
starting_frame=[0],
format='tiff',
starting_time=0.0,
rate=1.0)
nwbfile.add_acquisition(self.image_series)
mod = nwbfile.create_processing_module(
'ophys', 'contains optical physiology processed data')
img_seg = ImageSegmentation()
mod.add(img_seg)
ps = img_seg.create_plane_segmentation(
'output from segmenting my favorite imaging plane', imaging_plane,
'my_planeseg', self.image_series)
w, h = 3, 3
pix_mask1 = [(0, 0, 1.1), (1, 1, 1.2), (2, 2, 1.3)]
vox_mask1 = [(0, 0, 0, 1.1), (1, 1, 1, 1.2), (2, 2, 2, 1.3)]
img_mask1 = [[0.0 for x in range(w)] for y in range(h)]
img_mask1[0][0] = 1.1
img_mask1[1][1] = 1.2
img_mask1[2][2] = 1.3
ps.add_roi(pixel_mask=pix_mask1,
image_mask=img_mask1,
voxel_mask=vox_mask1)
pix_mask2 = [(0, 0, 2.1), (1, 1, 2.2)]
vox_mask2 = [(0, 0, 0, 2.1), (1, 1, 1, 2.2)]
img_mask2 = [[0.0 for x in range(w)] for y in range(h)]
img_mask2[0][0] = 2.1
img_mask2[1][1] = 2.2
ps.add_roi(pixel_mask=pix_mask2,
image_mask=img_mask2,
voxel_mask=vox_mask2)
fl = Fluorescence()
mod.add(fl)
rt_region = ps.create_roi_table_region('the first of two ROIs',
region=[0])
data = np.array([0., 1., 2., 3., 4., 5., 6., 7., 8.,
9.]).reshape(10, 1)
timestamps = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
rrs = fl.create_roi_response_series('my_rrs',
data,
rt_region,
unit='lumens',
timestamps=timestamps)
self.df_over_f = DfOverF(rrs)
####################
# Adding two-photon image data
# ----------------------------
#
# Now that you have your :py:class:`~pynwb.ophys.ImagingPlane`, you can create a
# :py:class:`~pynwb.ophys.TwoPhotonSeries` - the class representing two photon imaging data.
#
# From here you have two options. The first option is to supply the image data to PyNWB, using the `data` argument.
# The other option is the provide a path the images. These two options have trade-offs, so it is worth spending time
# considering how you want to store this data [#]_.
image_series = TwoPhotonSeries(name='test_iS',
dimension=[2],
external_file=['images.tiff'],
imaging_plane=imaging_plane,
starting_frame=[0],
format='tiff',
starting_time=0.0,
rate=1.0)
nwbfile.add_acquisition(image_series)
####################
# Storing image segmentation output
# ---------------------------------
#
# Now that the raw data is stored, you can add the image segmentation results. This is done with the
# :py:class:`~pynwb.ophys.ImageSegmentation` data interface. This class has the ability to store segmentation
# from one or more imaging planes; hence the :py:class:`~pynwb.ophys.PlaneSegmentation` class.
mod = nwbfile.create_processing_module('my_ca_imaging_module',
'example data module')
optical_channel = OpticalChannel('my channel', 'description', np.nan)
imaging_plane = nwbfile.create_imaging_plane('my_imgpln', optical_channel,
'description', 'imaging_device_1',
600., '2.718', 'GFP',
'my favorite brain location', [],
4.0, 'manifold unit',
'A frame to refer to')
images_path = os.path.join(base_dir, 'm655_D11_S1.hdf5')
with File(images_path, 'r') as file:
image_series = TwoPhotonSeries(name='test_iS',
dimension=[2],
data=file['Data']['Images'][:],
imaging_plane=imaging_plane,
starting_frame=[0],
starting_time=0.0,
rate=5.0,
scan_line_rate=np.nan,
pmt_gain=np.nan)
nwbfile.add_acquisition(image_series)
mod = nwbfile.create_processing_module('rois', 'example data module')
img_seg = ImageSegmentation()
ps = img_seg.create_plane_segmentation('Ca2+ imaging example', imaging_plane,
'my_planeseg', image_series)
mod.add_data_interface(img_seg)
for i, img_mask in enumerate(zip(mat_data['cellImages'])):
pixel_mask = np.array(np.where(img_mask)).T
ps.add_roi(str(i), pixel_mask, img_mask)
# :width: 800
# :alt: two-photon series UML diagram
# :align: center
#
# .. only:: latex
#
# .. image:: ../../_static/TwoPhotonSeries.png
# :width: 800
# :alt: two-photon series UML diagram
# :align: center
#
# using internal data. this data will be stored inside the NWB file
image_series1 = TwoPhotonSeries(name='TwoPhotonSeries1',
data=np.ones((1000, 100, 100)),
imaging_plane=imaging_plane,
rate=1.0,
unit='normalized amplitude')
# using external data. only the file paths will be stored inside the NWB file
image_series2 = TwoPhotonSeries(name='TwoPhotonSeries2',
dimension=[100, 100],
external_file=['images.tiff'],
imaging_plane=imaging_plane,
starting_frame=[0],
format='external',
starting_time=0.0,
rate=1.0)
####################
# Since these two-photon data are raw, acquired data, we will add the
def conversion_function(source_paths,
f_nwb,
metadata,
add_raw=False,
add_processed=True,
add_behavior=True,
plot_rois=False):
"""
Copy data stored in a set of .npz files to a single NWB file.
Parameters
----------
source_paths : dict
Dictionary with paths to source files/directories. e.g.:
{'raw_data': {'type': 'file', 'path': ''},
'raw_info': {'type': 'file', 'path': ''}
'processed_data': {'type': 'file', 'path': ''},
'sparse_matrix': {'type': 'file', 'path': ''},
'ref_image',: {'type': 'file', 'path': ''}}
f_nwb : str
Path to output NWB file, e.g. 'my_file.nwb'.
metadata : dict
Metadata dictionary
add_raw : bool
Whether to convert raw data or not.
add_processed : bool
Whether to convert processed data or not.
add_behavior : bool
Whether to convert behavior data or not.
plot_rois : bool
Plot ROIs
"""
# Source files
file_raw = None
file_info = None
file_processed = None
file_sparse_matrix = None
file_reference_image = None
for k, v in source_paths.items():
if source_paths[k]['path'] != '':
fname = source_paths[k]['path']
if k == 'raw_data':
file_raw = h5py.File(fname, 'r')
if k == 'raw_info':
file_info = scipy.io.loadmat(fname,
struct_as_record=False,
squeeze_me=True)
if k == 'processed_data':
file_processed = np.load(fname)
if k == 'sparse_matrix':
file_sparse_matrix = np.load(fname)
if k == 'ref_image':
file_reference_image = np.load(fname)
# Initialize a NWB object
nwb = NWBFile(**metadata['NWBFile'])
# Create and add device
device = Device(name=metadata['Ophys']['Device'][0]['name'])
nwb.add_device(device)
# Creates one Imaging Plane for each channel
fs = 1. / (file_processed['time'][0][1] - file_processed['time'][0][0])
for meta_ip in metadata['Ophys']['ImagingPlane']:
# Optical channel
opt_ch = OpticalChannel(
name=meta_ip['optical_channel'][0]['name'],
description=meta_ip['optical_channel'][0]['description'],
emission_lambda=meta_ip['optical_channel'][0]['emission_lambda'])
nwb.create_imaging_plane(
name=meta_ip['name'],
optical_channel=opt_ch,
description=meta_ip['description'],
device=device,
excitation_lambda=meta_ip['excitation_lambda'],
imaging_rate=fs,
indicator=meta_ip['indicator'],
location=meta_ip['location'],
)
# Raw optical data
if add_raw:
print('Adding raw data...')
for meta_tps in metadata['Ophys']['TwoPhotonSeries']:
if meta_tps['name'][-1] == 'R':
raw_data = file_raw['R']
else:
raw_data = file_raw['Y']
def data_gen(data):
xl, yl, zl, tl = data.shape
chunk = 0
while chunk < tl:
val = data[:, :, :, chunk]
chunk += 1
print('adding data chunk: ', chunk)
yield val
xl, yl, zl, tl = raw_data.shape
tps_data = DataChunkIterator(data=data_gen(data=raw_data),
iter_axis=0,
maxshape=(tl, xl, yl, zl))
# Change dimensions from (X,Y,Z,T) in mat file to (T,X,Y,Z) nwb standard
#raw_data = np.moveaxis(raw_data, -1, 0)
tps = TwoPhotonSeries(
name=meta_tps['name'],
imaging_plane=nwb.imaging_planes[meta_tps['imaging_plane']],
data=tps_data,
rate=file_info['info'].daq.scanRate)
nwb.add_acquisition(tps)
# Processed data
if add_processed:
print('Adding processed data...')
ophys_module = ProcessingModule(
name='Ophys',
description='contains optical physiology processed data.',
)
nwb.add_processing_module(ophys_module)
# Create Image Segmentation compartment
img_seg = ImageSegmentation(
name=metadata['Ophys']['ImageSegmentation']['name'])
ophys_module.add(img_seg)
# Create plane segmentation and add ROIs
meta_ps = metadata['Ophys']['ImageSegmentation'][
'plane_segmentations'][0]
ps = img_seg.create_plane_segmentation(
name=meta_ps['name'],
description=meta_ps['description'],
imaging_plane=nwb.imaging_planes[meta_ps['imaging_plane']],
)
# Add ROIs
indices = file_sparse_matrix['indices']
indptr = file_sparse_matrix['indptr']
dims = np.squeeze(file_processed['dims'])
for start, stop in zip(indptr, indptr[1:]):
voxel_mask = make_voxel_mask(indices[start:stop], dims)
ps.add_roi(voxel_mask=voxel_mask)
# Visualize 3D voxel masks
if plot_rois:
plot_rois_function(plane_segmentation=ps, indptr=indptr)
# DFF measures
dff = DfOverF(name=metadata['Ophys']['DfOverF']['name'])
ophys_module.add(dff)
# create ROI regions
n_cells = file_processed['dFF'].shape[0]
roi_region = ps.create_roi_table_region(description='RoiTableRegion',
region=list(range(n_cells)))
# create ROI response series
dff_data = file_processed['dFF']
tt = file_processed['time'].ravel()
meta_rrs = metadata['Ophys']['DfOverF']['roi_response_series'][0]
meta_rrs['data'] = dff_data.T
meta_rrs['rois'] = roi_region
meta_rrs['timestamps'] = tt
dff.create_roi_response_series(**meta_rrs)
# Creates GrayscaleVolume containers and add a reference image
grayscale_volume = GrayscaleVolume(
name=metadata['Ophys']['GrayscaleVolume']['name'],
data=file_reference_image['im'])
ophys_module.add(grayscale_volume)
# Behavior data
if add_behavior:
print('Adding behavior data...')
# Ball motion
behavior_mod = nwb.create_processing_module(
name='Behavior',
description='holds processed behavior data',
)
meta_ts = metadata['Behavior']['TimeSeries'][0]
meta_ts['data'] = file_processed['ball'].ravel()
tt = file_processed['time'].ravel()
meta_ts['timestamps'] = tt
behavior_ts = TimeSeries(**meta_ts)
behavior_mod.add(behavior_ts)
# Re-arranges spatial data of body-points positions tracking
pos = file_processed['dlc']
n_points = 8
pos_reshaped = pos.reshape(
(-1, n_points, 3)) # dims=(nSamples,n_points,3)
# Creates a Position object and add one SpatialSeries for each body-point position
position = Position()
for i in range(n_points):
position.create_spatial_series(
name='SpatialSeries_' + str(i),
data=pos_reshaped[:, i, :],
timestamps=tt,
reference_frame=
'Description defining what the zero-position is.',
conversion=np.nan)
behavior_mod.add(position)
# Trial times
trialFlag = file_processed['trialFlag'].ravel()
trial_inds = np.hstack(
(0, np.where(np.diff(trialFlag))[0], trialFlag.shape[0] - 1))
trial_times = tt[trial_inds]
for start, stop in zip(trial_times, trial_times[1:]):
nwb.add_trial(start_time=start, stop_time=stop)
# Saves to NWB file
with NWBHDF5IO(f_nwb, mode='w') as io:
io.write(nwb)
print('NWB file saved with size: ', os.stat(f_nwb).st_size / 1e6, ' mb')
def save(self,
file_name,
to32=True,
order='F',
imagej=False,
bigtiff=True,
excitation_lambda=488.0,
compress=0,
q_max=99.75,
q_min=1,
var_name_hdf5='mov',
sess_desc='some_description',
identifier='some identifier',
imaging_plane_description='some imaging plane description',
emission_lambda=520.0,
indicator='OGB-1',
location='brain',
starting_time=0.,
experimenter='Dr Who',
lab_name=None,
institution=None,
experiment_description='Experiment Description',
session_id='Session ID'):
"""
Save the timeseries in single precision. Supported formats include
TIFF, NPZ, AVI, MAT, HDF5/H5, MMAP, and NWB
Args:
file_name: str
name of file. Possible formats are tif, avi, npz, mmap and hdf5
to32: Bool
whether to transform to 32 bits
order: 'F' or 'C'
C or Fortran order
var_name_hdf5: str
Name of hdf5 file subdirectory
q_max, q_min: float in [0, 100]
percentile for maximum/minimum clipping value if saving as avi
(If set to None, no automatic scaling to the dynamic range [0, 255] is performed)
Raises:
Exception 'Extension Unknown'
"""
name, extension = os.path.splitext(file_name)[:2]
extension = extension.lower()
logging.debug("Parsing extension " + str(extension))
if extension in ['.tif', '.tiff', '.btf']:
with tifffile.TiffWriter(file_name, bigtiff=bigtiff,
imagej=imagej) as tif:
for i in range(self.shape[0]):
if i % 200 == 0:
logging.debug(str(i) + ' frames saved')
curfr = self[i].copy()
if to32 and not ('float32' in str(self.dtype)):
curfr = curfr.astype(np.float32)
tif.save(curfr, compress=compress)
elif extension == '.npz':
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
np.savez(file_name,
input_arr=input_arr,
start_time=self.start_time,
fr=self.fr,
meta_data=self.meta_data,
file_name=self.file_name)
elif extension == '.avi':
codec = None
try:
codec = cv2.FOURCC('I', 'Y', 'U', 'V')
except AttributeError:
codec = cv2.VideoWriter_fourcc(*'IYUV')
if q_max is None or q_min is None:
data = self.astype(np.uint8)
else:
if q_max < 100:
maxmov = np.nanpercentile(self[::max(1,
len(self) // 100)],
q_max)
else:
maxmov = np.nanmax(self)
if q_min > 0:
minmov = np.nanpercentile(self[::max(1,
len(self) // 100)],
q_min)
else:
minmov = np.nanmin(self)
data = 255 * (self - minmov) / (maxmov - minmov)
np.clip(data, 0, 255, data)
data = data.astype(np.uint8)
y, x = data[0].shape
vw = cv2.VideoWriter(file_name,
codec,
self.fr, (x, y),
isColor=True)
for d in data:
vw.write(cv2.cvtColor(d, cv2.COLOR_GRAY2BGR))
vw.release()
elif extension == '.mat':
if self.file_name[0] is not None:
f_name = self.file_name
else:
f_name = ''
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
if self.meta_data[0] is None:
savemat(
file_name, {
'input_arr': np.rollaxis(input_arr, axis=0, start=3),
'start_time': self.start_time,
'fr': self.fr,
'meta_data': [],
'file_name': f_name
})
else:
savemat(
file_name, {
'input_arr': np.rollaxis(input_arr, axis=0, start=3),
'start_time': self.start_time,
'fr': self.fr,
'meta_data': self.meta_data,
'file_name': f_name
})
elif extension in ('.hdf5', '.h5'):
with h5py.File(file_name, "w") as f:
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
dset = f.create_dataset(var_name_hdf5, data=input_arr)
dset.attrs["fr"] = self.fr
dset.attrs["start_time"] = self.start_time
try:
dset.attrs["file_name"] = [
a.encode('utf8') for a in self.file_name
]
except:
logging.warning('No file saved')
if self.meta_data[0] is not None:
logging.debug("Metadata for saved file: " +
str(self.meta_data))
dset.attrs["meta_data"] = cpk.dumps(self.meta_data)
elif extension == '.mmap':
base_name = name
T = self.shape[0]
dims = self.shape[1:]
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
input_arr = np.transpose(input_arr,
list(range(1,
len(dims) + 1)) + [0])
input_arr = np.reshape(input_arr, (np.prod(dims), T), order='F')
fname_tot = memmap_frames_filename(base_name, dims, T, order)
fname_tot = os.path.join(os.path.split(file_name)[0], fname_tot)
big_mov = np.memmap(fname_tot,
mode='w+',
dtype=np.float32,
shape=(np.uint64(np.prod(dims)), np.uint64(T)),
order=order)
big_mov[:] = np.asarray(input_arr, dtype=np.float32)
big_mov.flush()
del big_mov, input_arr
return fname_tot
elif extension == '.nwb':
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
# Create NWB file
nwbfile = NWBFile(sess_desc,
identifier,
datetime.now(tzlocal()),
experimenter=experimenter,
lab=lab_name,
institution=institution,
experiment_description=experiment_description,
session_id=session_id)
# Get the device
device = Device('imaging_device')
nwbfile.add_device(device)
# OpticalChannel
optical_channel = OpticalChannel('OpticalChannel',
'main optical channel',
emission_lambda=emission_lambda)
imaging_plane = nwbfile.create_imaging_plane(
name='ImagingPlane',
optical_channel=optical_channel,
description=imaging_plane_description,
device=device,
excitation_lambda=excitation_lambda,
imaging_rate=self.fr,
indicator=indicator,
location=location)
# Images
image_series = TwoPhotonSeries(name=var_name_hdf5,
dimension=self.shape[1:],
data=input_arr,
imaging_plane=imaging_plane,
starting_frame=[0],
starting_time=starting_time,
rate=self.fr)
nwbfile.add_acquisition(image_series)
with NWBHDF5IO(file_name, 'w') as io:
io.write(nwbfile)
return file_name
else:
logging.error("Extension " + str(extension) + " unknown")
raise Exception('Extension Unknown')
def add_ophys_processing_from_suite2p(save_folder, nwbfile, xml,
device=None,
optical_channel=None,
imaging_plane=None,
image_series=None):
"""
adapted from suite2p/suite2p/io/nwb.py "save_nwb" function
"""
plane_folders = natsorted([ f.path for f in os.scandir(save_folder) if f.is_dir() and f.name[:5]=='plane'])
OPS = [np.load(os.path.join(f, 'ops.npy'), allow_pickle=True).item() for f in plane_folders]
if len(OPS)>1:
multiplane, nplanes = True, len(plane_folders)
pData_folder = os.path.join(save_folder, 'combined') # processed data folder -> using the "combined output from suite2p"
else:
multiplane, nplanes = False, 1
pData_folder = os.path.join(save_folder, 'plane0') # processed data folder
# find time sampling per plane
functional_chan = ('Ch1' if len(xml['Ch1']['relativeTime'])>1 else 'Ch2') # functional channel is one of the two !!
CaImaging_timestamps = xml[functional_chan]['relativeTime']+float(xml['settings']['framePeriod'])/2.
ops = np.load(os.path.join(pData_folder, 'ops.npy'), allow_pickle=True).item()
if device is None:
device = nwbfile.create_device(
name='Microscope',
description='My two-photon microscope',
manufacturer='The best microscope manufacturer')
if optical_channel is None:
optical_channel = OpticalChannel(
name='OpticalChannel',
description='an optical channel',
emission_lambda=500.)
if imaging_plane is None:
imaging_plane = nwbfile.create_imaging_plane(
name='ImagingPlane',
optical_channel=optical_channel,
imaging_rate=ops['fs'],
description='standard',
device=device,
excitation_lambda=600.,
indicator='GCaMP',
location='V1',
grid_spacing=([2,2,30] if multiplane else [2,2]),
grid_spacing_unit='microns')
if image_series is None:
# link to external data
image_series = TwoPhotonSeries(
name='TwoPhotonSeries',
dimension=[ops['Ly'], ops['Lx']],
external_file=(ops['filelist'] if 'filelist' in ops else ['']),
imaging_plane=imaging_plane,
starting_frame=[0],
format='external',
starting_time=0.0,
rate=ops['fs'] * ops['nplanes']
)
nwbfile.add_acquisition(image_series) # otherwise, were added
# processing
img_seg = ImageSegmentation()
ps = img_seg.create_plane_segmentation(
name='PlaneSegmentation',
description='suite2p output',
imaging_plane=imaging_plane,
reference_images=image_series
)
ophys_module = nwbfile.create_processing_module(
name='ophys',
description='optical physiology processed data\n TSeries-folder=%s' % save_folder)
ophys_module.add(img_seg)
file_strs = ['F.npy', 'Fneu.npy', 'spks.npy']
traces = []
iscell = np.load(os.path.join(pData_folder, 'iscell.npy')).astype(bool)
if ops['nchannels']>1:
if os.path.isfile(os.path.join(pData_folder, 'redcell_manual.npy')):
redcell = np.load(os.path.join(pData_folder, 'redcell_manual.npy'))[iscell[:,0], :]
else:
print('\n'+30*'--')
print(' /!\ no file found for the manual labelling of red cells (generate it with the red-cell labelling GUI) /!\ ')
print(' /!\ taking the raw suit2p output with the classifier settings /!\ ')
print('\n'+30*'--')
redcell = np.load(os.path.join(pData_folder, 'redcell.npy'))[iscell[:,0], :]
for fstr in file_strs:
traces.append(np.load(os.path.join(pData_folder, fstr))[iscell[:,0], :])
stat = np.load(os.path.join(pData_folder, 'stat.npy'), allow_pickle=True)
ncells = np.sum(iscell[:,0])
plane_ID = np.zeros(ncells)
for n in np.arange(ncells):
pixel_mask = np.array([stat[iscell[:,0]][n]['ypix'], stat[iscell[:,0]][n]['xpix'],
stat[iscell[:,0]][n]['lam']])
ps.add_roi(pixel_mask=pixel_mask.T)
if 'iplane' in stat[0]:
plane_ID[n] = stat[iscell[:,0]][n]['iplane']
if ops['nchannels']>1:
ps.add_column('redcell', 'two columns - redcell & probcell', redcell)
ps.add_column('plane', 'one column - plane ID', plane_ID)
rt_region = ps.create_roi_table_region(
region=list(np.arange(0, ncells)),
description='all ROIs')
# FLUORESCENCE (all are required)
file_strs = ['F.npy', 'Fneu.npy', 'spks.npy']
name_strs = ['Fluorescence', 'Neuropil', 'Deconvolved']
for i, (fstr,nstr) in enumerate(zip(file_strs, name_strs)):
roi_resp_series = RoiResponseSeries(
name=nstr,
data=traces[i],
rois=rt_region,
unit='lumens',
timestamps=CaImaging_timestamps[::nplanes]) # ideally should be shifted for each ROI depending on the plane...
fl = Fluorescence(roi_response_series=roi_resp_series, name=nstr)
ophys_module.add(fl)
# BACKGROUNDS
# (meanImg, Vcorr and max_proj are REQUIRED)
bg_strs = ['meanImg', 'meanImgE', 'Vcorr', 'max_proj', 'meanImg_chan2']
nplanes = ops['nplanes']
for iplane in range(nplanes):
images = Images('Backgrounds_%d'%iplane)
for bstr in bg_strs:
if bstr in ops:
if bstr=='Vcorr' or bstr=='max_proj':
img = np.zeros((ops['Ly'], ops['Lx']), np.float32)
img[ops['yrange'][0]:ops['yrange'][-1],
ops['xrange'][0]:ops['xrange'][-1]] = ops[bstr]
else:
img = ops[bstr]
images.add_image(GrayscaleImage(name=bstr, data=img))
ophys_module.add(images)
