def build_Ca_filelist(folder):
CA_FILES = {'Bruker_folder':[], 'Bruker_file':[],
'date':[], 'protocol':[],'StartTimeString':[],
'StartTime':[], 'EndTime':[], 'absoluteTime':[]}
for bdf in get_TSeries_folders(folder):
fn = get_files_with_extension(bdf, extension='.xml')[0]
try:
xml = bruker_xml_parser(fn)
if len(xml['Ch1']['relativeTime'])>0:
CA_FILES['date'].append(stringdatetime_to_date(xml['date']))
CA_FILES['Bruker_folder'].append(bdf)
CA_FILES['Bruker_file'].append(fn)
CA_FILES['StartTimeString'].append(xml['StartTime'])
start = StartTime_to_day_seconds(xml['StartTime'])
CA_FILES['StartTime'].append(start+xml['Ch1']['absoluteTime'][0])
CA_FILES['EndTime'].append(start+xml['Ch1']['absoluteTime'][-1])
CA_FILES['protocol'].append('')
except BaseException as e:
print(e)
print(100*'-')
print('Problem with file: "%s"' % fn)
print(100*'-')
return CA_FILES
def build_suite2p_options(folder, settings_dict):
xml_file = os.path.join(folder,
os.path.join(folder.split('/')[-1] + '.xml'))
bruker_data = bruker_xml_parser(xml_file)
ops = ops0.copy()
# acquisition frequency
ops['fs'] = 1. / float(bruker_data['settings']['framePeriod'])
# hints for the size of the ROI
um_per_pixel = float(bruker_data['settings']['micronsPerPixel']['XAxis'])
ops['diameter'] = int(settings_dict['cell_diameter'] /
um_per_pixel) # in pixels (int 20um)
ops['spatial_scale'] = int(settings_dict['cell_diameter'] / 6 /
um_per_pixel)
# all other keys here
for key in settings_dict:
if key in ops:
ops[key] = settings_dict[key]
db = build_db(folder)
for key in [
'data_path', 'subfolders', 'save_path0', 'fast_disk',
'input_format'
]:
ops[key] = db[key]
np.save(os.path.join(folder, 'db.npy'), db)
np.save(os.path.join(folder, 'ops.npy'), ops)
def load_data(self, folder):
if folder != '':
self.folder = folder
xml_file = os.path.join(
folder,
get_files_with_extension(self.folder, extension='.xml')[0])
self.bruker_data = bruker_xml_parser(xml_file)
self.cframe = 0
self.nframes = len(
self.bruker_data[self.channelB.currentText()]['tifFile'])
self.display_frames()
self.ROIS = []
if os.path.isfile(os.path.join(self.folder, 'cells.npy')):
cells = np.load(os.path.join(self.folder, 'cells.npy'),
allow_pickle=True).item()
for x, y, z in zip(cells['x'], cells['y'], cells['z']):
self.ROIS.append(cellROI(depth=z, pos=(x, y), parent=self))
else:
print('"%s" is not a valid folder' % folder)
ops['spatial_scale'] = int(settings_dict['cell_diameter'] / 6 /
um_per_pixel)
# all other keys here
for key in settings_dict:
if key in ops:
ops[key] = settings_dict[key]
db = build_db(folder)
for key in [
'data_path', 'subfolders', 'save_path0', 'fast_disk',
'input_format'
]:
ops[key] = db[key]
np.save(os.path.join(folder, 'db.npy'), db)
np.save(os.path.join(folder, 'ops.npy'), ops)
if __name__ == '__main__':
folder = sys.argv[
-1] # '/media/yann/Yann/2020_11_10/TSeries-11102020-1605-016'
xml_file = os.path.join(folder,
os.path.join(folder.split('/')[-1] + '.xml'))
bruker_data = bruker_xml_parser(xml_file)
# print(freq)
# print(fn.split(os.path.sep))
def add_ophys(nwbfile,
args,
metadata=None,
with_raw_CaImaging=True,
with_processed_CaImaging=True,
Ca_Imaging_options={
'Suite2P-binary-filename': 'data.bin',
'plane': 0
}):
if metadata is None:
metadata = ast.literal_eval(nwbfile.session_description)
try:
CaFn = get_files_with_extension(
args.CaImaging_folder, extension='.xml')[0] # get Tseries metadata
except BaseException as be:
print(be)
print('\n /!\ Problem with the CA-IMAGING data in %s /!\ ' %
args.datafolder)
raise Exception
xml = bruker_xml_parser(CaFn) # metadata
onset = (metadata['STEP_FOR_CA_IMAGING_TRIGGER']['onset']
if 'STEP_FOR_CA_IMAGING_TRIGGER' in metadata else 0)
CaImaging_timestamps = onset+xml['Ch1']['relativeTime']+\
float(xml['settings']['framePeriod'])/2. # in the middle in-between two time stamps
device = pynwb.ophys.Device('Imaging device with settings: \n %s' %
str(xml['settings'])) # TO BE FILLED
nwbfile.add_device(device)
optical_channel = pynwb.ophys.OpticalChannel(
'excitation_channel 1', 'Excitation 1',
float(xml['settings']['laserWavelength']['Excitation 1']))
imaging_plane = nwbfile.create_imaging_plane(
'my_imgpln',
optical_channel,
description='Depth=%.1f[um]' %
float(xml['settings']['positionCurrent']['ZAxis']),
device=device,
excitation_lambda=float(
xml['settings']['laserWavelength']['Excitation 1']),
imaging_rate=1. / float(xml['settings']['framePeriod']),
indicator='GCamp',
location='V1',
# reference_frame='A frame to refer to',
grid_spacing=(float(xml['settings']['micronsPerPixel']['YAxis']),
float(xml['settings']['micronsPerPixel']['XAxis'])))
Ca_data = None
if with_raw_CaImaging:
if args.verbose:
print('=> Storing Calcium Imaging data [...]')
Ca_data = BinaryFile(
Ly=int(xml['settings']['linesPerFrame']),
Lx=int(xml['settings']['pixelsPerLine']),
read_filename=os.path.join(
args.CaImaging_folder, 'suite2p',
'plane%i' % Ca_Imaging_options['plane'],
Ca_Imaging_options['Suite2P-binary-filename']))
CA_SUBSAMPLING = build_subsampling_from_freq(\
subsampled_freq=args.CaImaging_frame_sampling,
original_freq=1./float(xml['settings']['framePeriod']),
N=Ca_data.shape[0], Nmin=3)
if args.CaImaging_frame_sampling > 0:
dI = int(1. / args.CaImaging_frame_sampling /
float(xml['settings']['framePeriod']))
else:
dI = 1
def Ca_frame_generator():
for i in CA_SUBSAMPLING:
yield Ca_data.data[i:i + dI, :, :].mean(axis=0).astype(
np.uint8)
Ca_dataI = DataChunkIterator(data=Ca_frame_generator(),
maxshape=(None, Ca_data.shape[1],
Ca_data.shape[2]),
dtype=np.dtype(np.uint8))
if args.compression > 0:
Ca_dataC = H5DataIO(
data=Ca_dataI, # with COMPRESSION
compression='gzip',
compression_opts=args.compression)
image_series = pynwb.ophys.TwoPhotonSeries(
name='CaImaging-TimeSeries',
dimension=[2],
data=Ca_dataC,
imaging_plane=imaging_plane,
unit='s',
timestamps=CaImaging_timestamps[CA_SUBSAMPLING])
else:
image_series = pynwb.ophys.TwoPhotonSeries(
name='CaImaging-TimeSeries',
dimension=[2],
data=Ca_dataI,
# data = Ca_data.data[:].astype(np.uint8),
imaging_plane=imaging_plane,
unit='s',
timestamps=CaImaging_timestamps[CA_SUBSAMPLING])
else:
image_series = pynwb.ophys.TwoPhotonSeries(name='CaImaging-TimeSeries',
dimension=[2],
data=np.ones((2, 2, 2)),
imaging_plane=imaging_plane,
unit='s',
timestamps=1. *
np.arange(2))
nwbfile.add_acquisition(image_series)
if with_processed_CaImaging and os.path.isdir(
os.path.join(args.CaImaging_folder, 'suite2p')):
print('=> Adding the suite2p processing [...]')
add_ophys_processing_from_suite2p(
os.path.join(args.CaImaging_folder, 'suite2p'),
nwbfile,
CaImaging_timestamps,
device=device,
optical_channel=optical_channel,
imaging_plane=imaging_plane,
image_series=image_series) # ADD UPDATE OF starting_time
elif with_processed_CaImaging:
print('\n /!\ no "suite2p" folder found in "%s" /!\ ' %
args.CaImaging_folder)
return Ca_data
def add_ophys(nwbfile,
args,
metadata=None,
with_raw_CaImaging=True,
with_processed_CaImaging=True,
Ca_Imaging_options={
'Suite2P-binary-filename': 'data.bin',
'plane': 0
}):
#########################################
########## Loading metadata ############
#########################################
if metadata is None:
metadata = ast.literal_eval(nwbfile.session_description)
try:
CaFn = get_files_with_extension(
args.CaImaging_folder, extension='.xml')[0] # get Tseries metadata
except BaseException as be:
print(be)
print('\n /!\ Problem with the CA-IMAGING data in %s /!\ ' %
args.datafolder)
raise Exception
xml = bruker_xml_parser(CaFn) # metadata
##################################################
########## setup-specific quantities ############
##################################################
if 'A1-2P' in metadata['Rig']:
functional_chan = 'Ch2' # green channel is channel 2 downstairs
laser_key = 'Laser'
Depth = float(xml['settings']['positionCurrent']['ZAxis']['Z Focus']
[0]) # center depth only !
else:
functional_chan = 'Ch1'
laser_key = 'Excitation 1'
Depth = float(xml['settings']['positionCurrent']['ZAxis'])
##################################################
########## setup-specific quantities ############
##################################################
device = pynwb.ophys.Device('Imaging device with settings: \n %s' %
str(xml['settings'])) # TO BE FILLED
nwbfile.add_device(device)
optical_channel = pynwb.ophys.OpticalChannel(
'excitation_channel 1', laser_key,
float(xml['settings']['laserWavelength'][laser_key]))
multiplane = (True if len(np.unique(xml['depth_shift'])) > 1 else False)
if not multiplane:
imaging_plane = nwbfile.create_imaging_plane(
'my_imgpln',
optical_channel,
description='Depth=%.1f[um]' %
(float(metadata['Z-sign-correction-for-rig']) * Depth),
device=device,
excitation_lambda=float(
xml['settings']['laserWavelength'][laser_key]),
imaging_rate=1. / float(xml['settings']['framePeriod']),
indicator='GCamp',
location='V1', # ADD METADATA HERE
# reference_frame='A frame to refer to',
grid_spacing=(float(xml['settings']['micronsPerPixel']['YAxis']),
float(xml['settings']['micronsPerPixel']['XAxis'])))
else:
# DESCRIBE THE MULTIPLANES HERE !!!!
imaging_plane = nwbfile.create_imaging_plane(
'my_imgpln',
optical_channel,
description='Depth=%.1f[um]' %
(float(metadata['Z-sign-correction-for-rig']) * Depth),
device=device,
excitation_lambda=float(
xml['settings']['laserWavelength'][laser_key]),
imaging_rate=1. / float(xml['settings']['framePeriod']),
indicator='GCamp',
location='V1', # ADD METADATA HERE
# reference_frame='A frame to refer to',
grid_spacing=(float(xml['settings']['micronsPerPixel']['YAxis']),
float(xml['settings']['micronsPerPixel']['XAxis'])))
########################################
##### --- DEPRECATED to be fixed ... ##
########################################
Ca_data = None
# if with_raw_CaImaging:
# if args.verbose:
# print('=> Storing Calcium Imaging data [...]')
# Ca_data = BinaryFile(Ly=int(xml['settings']['linesPerFrame']),
# Lx=int(xml['settings']['pixelsPerLine']),
# read_filename=os.path.join(args.CaImaging_folder,
# 'suite2p', 'plane%i' % Ca_Imaging_options['plane'],
# Ca_Imaging_options['Suite2P-binary-filename']))
# CA_SUBSAMPLING = build_subsampling_from_freq(\
# subsampled_freq=args.CaImaging_frame_sampling,
# original_freq=1./float(xml['settings']['framePeriod']),
# N=Ca_data.shape[0], Nmin=3)
# if args.CaImaging_frame_sampling>0:
# dI = int(1./args.CaImaging_frame_sampling/float(xml['settings']['framePeriod']))
# else:
# dI = 1
# def Ca_frame_generator():
# for i in CA_SUBSAMPLING:
# yield Ca_data.data[i:i+dI, :, :].mean(axis=0).astype(np.uint8)
# Ca_dataI = DataChunkIterator(data=Ca_frame_generator(),
# maxshape=(None, Ca_data.shape[1], Ca_data.shape[2]),
# dtype=np.dtype(np.uint8))
# if args.compression>0:
# Ca_dataC = H5DataIO(data=Ca_dataI, # with COMPRESSION
# compression='gzip',
# compression_opts=args.compression)
# image_series = pynwb.ophys.TwoPhotonSeries(name='CaImaging-TimeSeries',
# dimension=[2],
# data = Ca_dataC,
# imaging_plane=imaging_plane,
# unit='s',
# timestamps = CaImaging_timestamps[CA_SUBSAMPLING])
# else:
# image_series = pynwb.ophys.TwoPhotonSeries(name='CaImaging-TimeSeries',
# dimension=[2],
# data = Ca_dataI,
# # data = Ca_data.data[:].astype(np.uint8),
# imaging_plane=imaging_plane,
# unit='s',
# timestamps = CaImaging_timestamps[CA_SUBSAMPLING])
# else:
# image_series = pynwb.ophys.TwoPhotonSeries(name='CaImaging-TimeSeries',
# dimension=[2],
# data = np.ones((2,2,2)),
# imaging_plane=imaging_plane,
# unit='s',
# timestamps = 1.*np.arange(2))
# just a dummy version for now
image_series = pynwb.ophys.TwoPhotonSeries(
name='CaImaging-TimeSeries\n raw-data-folder=%s' %
args.CaImaging_folder.replace('/', '_'),
dimension=[2],
data=np.ones((2, 2, 2)),
imaging_plane=imaging_plane,
unit='s',
timestamps=1. * np.arange(2)) # TEMPORARY
nwbfile.add_acquisition(image_series)
if with_processed_CaImaging and os.path.isdir(
os.path.join(args.CaImaging_folder, 'suite2p')):
print('=> Adding the suite2p processing [...]')
add_ophys_processing_from_suite2p(
os.path.join(args.CaImaging_folder, 'suite2p'),
nwbfile,
xml,
device=device,
optical_channel=optical_channel,
imaging_plane=imaging_plane,
image_series=image_series) # ADD UPDATE OF starting_time
elif with_processed_CaImaging:
print('\n /!\ no "suite2p" folder found in "%s" /!\ ' %
args.CaImaging_folder)
return Ca_data