def add_running_speed_to_nwbfile(nwbfile, running_speed, name='speed', unit='cm/s'):
''' Adds running speed data to an NWBFile as a timeseries in acquisition
Parameters
----------
nwbfile : pynwb.NWBFile
File to which runnign speeds will be written
running_speed : RunningSpeed
Contains attributes 'values' and 'timestamps'
name : str, optional
used as name of timeseries object
unit : str, optional
SI units of running speed values
Returns
-------
nwbfile : pynwb.NWBFile
'''
running_speed_series = pynwb.base.TimeSeries(
name=name,
data=running_speed.values,
timestamps=running_speed.timestamps,
unit=unit
)
running_mod = ProcessingModule('running', 'Running speed processing module')
nwbfile.add_processing_module(running_mod)
running_mod.add_data_interface(running_speed_series)
return nwbfile
def add_image(nwbfile, image_data, image_name, module_name, module_description, image_api=None):
description = '{} image at pixels/cm resolution'.format(image_name)
if image_api is None:
image_api = ImageApi
if isinstance(image_data, sitk.Image):
data, spacing, unit = ImageApi.deserialize(image_data)
elif isinstance(image_data, Image):
data = image_data.data
spacing = image_data.spacing
unit = image_data.unit
else:
raise ValueError("Not a supported image_data type: {}".format(type(image_data)))
assert spacing[0] == spacing[1] and len(spacing) == 2 and unit == 'mm'
if module_name not in nwbfile.modules:
ophys_mod = ProcessingModule(module_name, module_description)
nwbfile.add_processing_module(ophys_mod)
else:
ophys_mod = nwbfile.modules[module_name]
image = GrayscaleImage(image_name, data, resolution=spacing[0] / 10, description=description)
if 'images' not in ophys_mod.containers:
images = Images(name='images')
ophys_mod.add_data_interface(images)
else:
images = ophys_mod['images']
images.add_image(image)
return nwbfile
def to_nwb(self, nwbfile: NWBFile) -> NWBFile:
# If there is no rewards data, do not
# write anything to the NWB file (this
# is expected for passive sessions)
if len(self.value['timestamps']) == 0:
return nwbfile
reward_volume_ts = TimeSeries(
name='volume',
data=self.value['volume'].values,
timestamps=self.value['timestamps'].values,
unit='mL'
)
autorewarded_ts = TimeSeries(
name='autorewarded',
data=self.value['autorewarded'].values,
timestamps=reward_volume_ts.timestamps,
unit='mL'
)
rewards_mod = ProcessingModule('rewards',
'Licking behavior processing module')
rewards_mod.add_data_interface(reward_volume_ts)
rewards_mod.add_data_interface(autorewarded_ts)
nwbfile.add_processing_module(rewards_mod)
return nwbfile
def to_nwb(self, nwbfile: NWBFile) -> NWBFile:
running_speed: pd.DataFrame = self.value
data = running_speed['speed'].values
timestamps = running_speed['timestamps'].values
if self._filtered:
data_interface_name = "speed"
else:
data_interface_name = "speed_unfiltered"
running_speed_series = TimeSeries(name=data_interface_name,
data=data,
timestamps=timestamps,
unit='cm/s')
if 'running' in nwbfile.processing:
running_mod = nwbfile.processing['running']
else:
running_mod = ProcessingModule('running',
'Running speed processing module')
nwbfile.add_processing_module(running_mod)
running_mod.add_data_interface(running_speed_series)
return nwbfile
def to_nwb(self, nwbfile: NWBFile) -> NWBFile:
stimulus_ts = TimeSeries(data=self._value,
name="timestamps",
timestamps=self._value,
unit="s")
stim_mod = ProcessingModule("stimulus", "Stimulus Times processing")
stim_mod.add_data_interface(stimulus_ts)
nwbfile.add_processing_module(stim_mod)
return nwbfile
def append_spike_times(input_nwb_path: PathLike,
sweep_spike_times: Dict[int, List[float]],
output_nwb_path: Optional[PathLike] = None):
"""
Appends spiketimes to an nwb2 file
Paramters
---------
input_nwb_path: location of input nwb file without spiketimes
spike_times: Dict of sweep_num: spiketimes
output_nwb_path: optional location to write new nwb file with
spiketimes, otherwise appends spiketimes to
input file
"""
# Copy to new location
if output_nwb_path and output_nwb_path != input_nwb_path:
shutil.copy(input_nwb_path, output_nwb_path)
nwb_path = output_nwb_path
else:
nwb_path = input_nwb_path
nwb_io = pynwb.NWBHDF5IO(nwb_path, mode='a', load_namespaces=True)
nwbfile = nwb_io.read()
spikes_module = "spikes"
# Add spikes only if not previously added
if spikes_module not in nwbfile.processing.keys():
spike_module = ProcessingModule(name=spikes_module,
description='detected spikes')
for sweep_num, spike_times in sweep_spike_times.items():
wrapped_spike_times = H5DataIO(data=np.asarray(spike_times),
compression=True)
ts = TimeSeries(timestamps=wrapped_spike_times,
unit='seconds',
data=wrapped_spike_times,
name=f"Sweep_{sweep_num}")
spike_module.add_data_interface(ts)
nwbfile.add_processing_module(spike_module)
nwb_io.write(nwbfile)
else:
raise ValueError("Cannot add spikes times to the nwb file: "
"spikes times already exist!")
nwb_io.close()
def add_spike_times(self, sweep_spike_times):
spike_module = ProcessingModule(name='spikes',
description='detected spikes')
for sweep_num, spike_times in sweep_spike_times.items():
ts = TimeSeries(timestamps=spike_times, name=f"Sweep_{sweep_num}")
spike_module.add_data_interface(ts)
self.nwbfile.add_processing_module(spike_module)
io = pynwb.NWBHDF5IO(self.nwb_file_name, 'w')
io.write(self.nwbfile)
io.close()
def add_stimulus_timestamps(nwbfile,
stimulus_timestamps,
module_name='stimulus'):
stimulus_ts = TimeSeries(data=stimulus_timestamps,
name='timestamps',
timestamps=stimulus_timestamps,
unit='s')
stim_mod = ProcessingModule(module_name, 'Stimulus Times processing')
nwbfile.add_processing_module(stim_mod)
stim_mod.add_data_interface(stimulus_ts)
return nwbfile
def add_running_speed_to_nwbfile(nwbfile,
running_speed,
name='speed',
unit='cm/s',
from_dataframe=False):
''' Adds running speed data to an NWBFile as a timeseries in acquisition
Parameters
----------
nwbfile : pynwb.NWBFile
File to which running speeds will be written
running_speed : Union[RunningSpeed, pd.DataFrame]
Either a RunningSpeed object or pandas DataFrame.
Contains attributes 'values' and 'timestamps'
name : str, optional
Used as name of timeseries object
unit : str, optional
SI units of running speed values
from_dataframe : bool, optional
Whether `running_speed` is a dataframe or not. Default is False.
Returns
-------
nwbfile : pynwb.NWBFile
'''
if from_dataframe:
data = running_speed['speed'].values
timestamps = running_speed['timestamps'].values
else:
data = running_speed.values
timestamps = running_speed.timestamps
running_speed_series = pynwb.base.TimeSeries(name=name,
data=data,
timestamps=timestamps,
unit=unit)
if 'running' in nwbfile.processing:
running_mod = nwbfile.processing['running']
else:
running_mod = ProcessingModule('running',
'Running speed processing module')
nwbfile.add_processing_module(running_mod)
running_mod.add_data_interface(running_speed_series)
return nwbfile
def to_nwb(self, nwbfile: NWBFile) -> NWBFile:
lick_timeseries = TimeSeries(
name='licks',
data=self.value['frame'].values,
timestamps=self.value['timestamps'].values,
description=('Timestamps and stimulus presentation '
'frame indices for lick events'),
unit='N/A')
# Add lick interface to nwb file, by way of a processing module:
licks_mod = ProcessingModule('licking',
'Licking behavior processing module')
licks_mod.add_data_interface(lick_timeseries)
nwbfile.add_processing_module(licks_mod)
return nwbfile
def add_licks(nwbfile, licks):
licks_event_series = TimeSeries(data=licks.time.values,
name='timestamps',
timestamps=licks.time.values,
unit='s')
# Add lick event timeseries to lick interface:
licks_interface = BehavioralEvents([licks_event_series], 'licks')
# Add lick interface to nwb file, by way of a processing module:
licks_mod = ProcessingModule('licking',
'Licking behavior processing module')
licks_mod.add_data_interface(licks_interface)
nwbfile.add_processing_module(licks_mod)
return nwbfile
def add_licks(nwbfile, licks):
lick_timeseries = TimeSeries(
name='licks',
data=licks.frame.values,
timestamps=licks.timestamps.values,
description=('Timestamps and stimulus presentation '
'frame indices for lick events'),
unit='N/A')
# Add lick interface to nwb file, by way of a processing module:
licks_mod = ProcessingModule('licking',
'Licking behavior processing module')
licks_mod.add_data_interface(lick_timeseries)
nwbfile.add_processing_module(licks_mod)
return nwbfile
def add_rewards(nwbfile, rewards_df):
reward_volume_ts = TimeSeries(name='volume',
data=rewards_df.volume.values,
timestamps=rewards_df['timestamps'].values,
unit='mL')
autorewarded_ts = TimeSeries(name='autorewarded',
data=rewards_df.autorewarded.values,
timestamps=reward_volume_ts.timestamps,
unit='mL')
rewards_mod = ProcessingModule('rewards',
'Licking behavior processing module')
rewards_mod.add_data_interface(reward_volume_ts)
rewards_mod.add_data_interface(autorewarded_ts)
nwbfile.add_processing_module(rewards_mod)
return nwbfile
def to_nwb(self, nwbfile: NWBFile) -> NWBFile:
reward_volume_ts = TimeSeries(
name='volume',
data=self.value['volume'].values,
timestamps=self.value['timestamps'].values,
unit='mL')
autorewarded_ts = TimeSeries(name='autorewarded',
data=self.value['autorewarded'].values,
timestamps=reward_volume_ts.timestamps,
unit='mL')
rewards_mod = ProcessingModule('rewards',
'Licking behavior processing module')
rewards_mod.add_data_interface(reward_volume_ts)
rewards_mod.add_data_interface(autorewarded_ts)
nwbfile.add_processing_module(rewards_mod)
return nwbfile
def add_running_acquisition_to_nwbfile(nwbfile,
running_acquisition_df: pd.DataFrame):
running_dx_series = TimeSeries(
name='dx',
data=running_acquisition_df['dx'].values,
timestamps=running_acquisition_df.index.values,
unit='cm',
description=(
'Running wheel angular change, computed during data collection'))
v_sig = TimeSeries(
name='v_sig',
data=running_acquisition_df['v_sig'].values,
timestamps=running_acquisition_df.index.values,
unit='V',
description='Voltage signal from the running wheel encoder')
v_in = TimeSeries(
name='v_in',
data=running_acquisition_df['v_in'].values,
timestamps=running_acquisition_df.index.values,
unit='V',
description=(
'The theoretical maximum voltage that the running wheel encoder '
'will reach prior to "wrapping". This should '
'theoretically be 5V (after crossing 5V goes to 0V, or '
'vice versa). In practice the encoder does not always '
'reach this value before wrapping, which can cause '
'transient spikes in speed at the voltage "wraps".'))
if 'running' in nwbfile.processing:
running_mod = nwbfile.processing['running']
else:
running_mod = ProcessingModule('running',
'Running speed processing module')
nwbfile.add_processing_module(running_mod)
running_mod.add_data_interface(running_dx_series)
nwbfile.add_acquisition(v_sig)
nwbfile.add_acquisition(v_in)
return nwbfile
def add_image_to_nwb(nwbfile: NWBFile, image_data: Image, image_name: str):
"""
Adds image given by image_data with name image_name to nwbfile
Parameters
----------
nwbfile
nwbfile to add image to
image_data
The image data
image_name
Image name
Returns
-------
None
"""
module_name = 'ophys'
description = '{} image at pixels/cm resolution'.format(image_name)
data, spacing, unit = image_data
assert spacing[0] == spacing[1] and len(spacing) == 2 and unit == 'mm'
if module_name not in nwbfile.processing:
ophys_mod = ProcessingModule(module_name, 'Ophys processing module')
nwbfile.add_processing_module(ophys_mod)
else:
ophys_mod = nwbfile.processing[module_name]
image = GrayscaleImage(image_name,
data,
resolution=spacing[0] / 10,
description=description)
if 'images' not in ophys_mod.containers:
images = Images(name='images')
ophys_mod.add_data_interface(images)
else:
images = ophys_mod['images']
images.add_image(image)
def add_motion_correction(nwbfile, motion_correction):
twop_module = nwbfile.modules['two_photon_imaging']
ophys_timestamps = twop_module.get_data_interface(
'dff').roi_response_series['traces'].timestamps
t1 = TimeSeries(name='x',
data=motion_correction['x'].values,
timestamps=ophys_timestamps,
unit='pixels')
t2 = TimeSeries(name='y',
data=motion_correction['y'].values,
timestamps=ophys_timestamps,
unit='pixels')
motion_module = ProcessingModule('motion_correction',
'Motion Correction processing module')
motion_module.add_data_interface(t1)
motion_module.add_data_interface(t2)
nwbfile.add_processing_module(motion_module)
def to_nwb(self, nwbfile: NWBFile) -> NWBFile:
# If there is no lick data, do not write
# anything to the NWB file (this is
# expected for passive sessions)
if len(self.value['frame']) == 0:
return nwbfile
lick_timeseries = TimeSeries(
name='licks',
data=self.value['frame'].values,
timestamps=self.value['timestamps'].values,
description=('Timestamps and stimulus presentation '
'frame indices for lick events'),
unit='N/A')
# Add lick interface to nwb file, by way of a processing module:
licks_mod = ProcessingModule('licking',
'Licking behavior processing module')
licks_mod.add_data_interface(lick_timeseries)
nwbfile.add_processing_module(licks_mod)
return nwbfile
def test_read_nwb_nwb_image_series_successfully(self):
device_1 = Device('device1')
device_2 = Device('device2')
mock_timestamps = [1, 2, 3]
mock_external_file = ['some file']
nwb_image_series = NwbImageSeries(name='NwbImageSeries1',
timestamps=mock_timestamps,
external_file=mock_external_file,
devices=[device_1, device_2])
behavioral_time_series = BehavioralEvents(name="BehavioralTimeSeries")
behavioral_time_series.add_timeseries(nwb_image_series)
processing_module = ProcessingModule(name='ProcessingModule',
description='')
processing_module.add_data_interface(behavioral_time_series)
self.nwb_file_content.add_processing_module(processing_module)
self.nwb_file_content.add_stimulus_template(nwb_image_series)
nwb_file_handler = NWBHDF5IO('nwb_image_series.nwb', mode='w')
nwb_file_handler.write(self.nwb_file_content)
nwb_file_handler.close()
self.assertTrue(os.path.exists('nwb_image_series.nwb'))
with pynwb.NWBHDF5IO('nwb_image_series.nwb', 'r',
load_namespaces=True) as nwb_file_handler:
nwb_file = nwb_file_handler.read()
self.assertContainerEqual(
nwb_file.stimulus_template['NwbImageSeries1'],
nwb_image_series)
self.assertContainerEqual(
nwb_file.processing['ProcessingModule'].data_interfaces[
'BehavioralTimeSeries'].time_series['NwbImageSeries1'],
nwb_image_series)
self.delete_nwb('nwb_image_series')
nwbfile.add_processing_module(added_mod)
####################
# You can add data to your processing module using the method
# :py:func:`~pynwb.base.ProcessingModule.add_data_interface`.
# Lets make another :py:class:`~pynwb.base.TimeSeries` and then add it to the
# :py:class:`~pynwb.base.ProcessingModule` we just added.
data = list(range(0, 100, 10))
timestamps = list(range(10))
mod_ts = TimeSeries('ts_for_mod',
'PyNWB tutorial',
data,
'SIunit',
timestamps=timestamps)
added_mod.add_data_interface(mod_ts)
####################
# .. _basic_epochs:
#
# Epochs
# ------
#
# Epochs can be added to an NWB file using the method :py:func:`~pynwb.file.NWBFile.create_epoch`.
# The first argument is a description of the epoch, the second and third argument are the start time
# and stop time, respectively. The fourth argument is one or more tags for labelling the epoch,
# and the fifth argument is a list of all the :py:class:`~pynwb.base.TimeSeries` that the epoch applies
# to.
nwbfile.create_epoch('the first epoch', 2.0, 4.0, ['first', 'example'],
[test_ts, mod_ts])
def no2nwb(NOData, session_use, subjects):
# Prepare the NO data that will be coverted to the NWB format
session = NOData.sessions[session_use]
events = NOData._get_event_data(session_use, experiment_type='All')
cell_ids = NOData.ls_cells(session_use)
experiment_id_learn = session['experiment_id_learn']
experiment_id_recog = session['experiment_id_recog']
task_descr = session['task_descr']
# Get the metadata for the subject
df_session = subjects[subjects['session_id'] == session_use]
print('session_use')
print(session_use)
print('age')
print(str(df_session['age'].values[0]))
print('epilepsy_diagnosis')
print(str(df_session['epilepsy_diagnosis'].values[0]))
nwb_subject = Subject(
age=str(df_session['age'].values[0]),
description=df_session['epilepsy_diagnosis'].values[0],
sex=df_session['sex'].values[0],
subject_id=df_session['subject_id'].values[0])
# Create the NWB file
nwbfile = NWBFile(
#source='https://datadryad.org/bitstream/handle/10255/dryad.163179/RecogMemory_MTL_release_v2.zip',
session_description='RecogMemory dataset session use 5' + session['session'],
identifier=session['session_id'],
session_start_time=datetime.datetime.now(),# TODO: need to check out the time for session start
file_create_date=datetime.datetime.now(),
experiment_description="learning: " + str(experiment_id_learn) + ", " + \
"recognition: " + \
str(experiment_id_recog),
subject=nwb_subject
)
# Add event and experiment_id acquisition
# event_ts = TimeSeries(name='events', source='NA', unit='NA', data=np.asarray(events[1].values),
# timestamps=np.asarray(events[0].values))
event_ts = TimeSeries(name='events',
unit='NA',
data=np.asarray(events[1].values),
timestamps=np.asarray(events[0].values))
# experiment_ids = TimeSeries(name='experiment_ids', source='NA', unit='NA', data=np.asarray(events[2]),
# timestamps=np.asarray(events[0].values))
experiment_ids = TimeSeries(name='experiment_ids',
unit='NA',
data=np.asarray(events[2]),
timestamps=np.asarray(events[0].values))
nwbfile.add_acquisition(event_ts)
nwbfile.add_acquisition(experiment_ids)
# Add stimuli to the NWB file2
# Get the first cell from the cell list
cell = NOData.pop_cell(session_use, NOData.ls_cells(session_use)[0])
trials = cell.trials
stimuli_recog_path = [trial.file_path_recog for trial in trials]
stimuli_learn_path = [trial.file_path_learn for trial in trials]
# Add stimuli recog
counter = 1
for path in stimuli_recog_path:
folders = path.split('\\')
path = os.path.join('./RecogMemory_MTL_release_v2', 'Stimuli',
folders[0], folders[1], folders[2])
img = cv2.imread(path)
name = 'stimuli_recog_' + str(counter)
stimulus_recog = ImageSeries(name=name,
data=img,
unit='NA',
format='',
timestamps=[0.0])
nwbfile.add_stimulus(stimulus_recog)
counter += 1
# Add stimuli learn
counter = 1
for path in stimuli_learn_path:
if path == 'NA':
continue
folders = path.split('\\')
path = os.path.join('./RecogMemory_MTL_release_v2', 'Stimuli',
folders[0], folders[1], folders[2])
img = cv2.imread(path)
name = 'stimuli_learn_' + str(counter)
stimulus_learn = ImageSeries(name=name,
data=img,
unit='NA',
format='',
timestamps=[0.0])
nwbfile.add_stimulus(stimulus_learn)
counter += 1
# Add epochs and trials: storing start and end times for a stimulus
# First extract the category ids and names that we need
# The metadata for each trials will be store in a trial table
cat_id_recog = [trial.category_recog for trial in trials]
cat_name_recog = [trial.category_name_recog for trial in trials]
cat_id_learn = [trial.category_learn for trial in trials]
cat_name_learn = [trial.category_name_learn for trial in trials]
# Extract the event timestamps
events_learn_stim_on = events[(events[2] == experiment_id_learn) &
(events[1] == NOData.markers['stimulus_on'])]
events_learn_stim_off = events[(events[2] == experiment_id_learn) & (
events[1] == NOData.markers['stimulus_off'])]
events_learn_delay1_off = events[(events[2] == experiment_id_learn) & (
events[1] == NOData.markers['delay1_off'])]
events_learn_delay2_off = events[(events[2] == experiment_id_learn) & (
events[1] == NOData.markers['delay2_off'])]
events_recog_stim_on = events[(events[2] == experiment_id_recog) &
(events[1] == NOData.markers['stimulus_on'])]
events_recog_stim_off = events[(events[2] == experiment_id_recog) & (
events[1] == NOData.markers['stimulus_off'])]
events_recog_delay1_off = events[(events[2] == experiment_id_recog) & (
events[1] == NOData.markers['delay1_off'])]
events_recog_delay2_off = events[(events[2] == experiment_id_recog) & (
events[1] == NOData.markers['delay2_off'])]
# Extract new_old label
new_old_recog = [trial.new_old_recog for trial in trials]
# Create the trial tables
nwbfile.add_trial_column('stim_on', 'the time when the stimulus is shown')
nwbfile.add_trial_column('stim_off', 'the time when the stimulus is off')
nwbfile.add_trial_column('delay1_off', 'the time when delay1 is off')
nwbfile.add_trial_column('delay2_off', 'the time when delay2 is off')
nwbfile.add_trial_column('stim_phase',
'learning/recognition phase during the trial')
nwbfile.add_trial_column('category_id', 'the category id of the stimulus')
nwbfile.add_trial_column('category_name',
'the category name of the stimulus')
nwbfile.add_trial_column('external_image_file',
'the file path to the stimulus')
nwbfile.add_trial_column('new_old_labels_recog',
'labels for new or old stimulus')
range_recog = np.amin([
len(events_recog_stim_on),
len(events_recog_stim_off),
len(events_recog_delay1_off),
len(events_recog_delay2_off)
])
range_learn = np.amin([
len(events_learn_stim_on),
len(events_learn_stim_off),
len(events_learn_delay1_off),
len(events_learn_delay2_off)
])
# Iterate the event list and add information into each epoch and trial table
for i in range(range_learn):
# nwbfile.create_epoch(start_time=events_learn_stim_on.iloc[i][0],
# stop_time=events_learn_stim_off.iloc[i][0],
# timeseries=[event_ts, experiment_ids],
# tags='stimulus_learn',
# description='learning phase stimulus')
# nwbfile.add_trial({'start': events_learn_stim_on.iloc[i][0],
# 'end': events_learn_delay2_off.iloc[i][0],
# 'stim_on': events_learn_stim_on.iloc[i][0],
# 'stim_off': events_learn_stim_off.iloc[i][0],
# 'delay1_off': events_learn_delay1_off.iloc[i][0],
# 'delay2_off': events_learn_delay2_off.iloc[i][0],
# 'stim_phase': 'learn',
# 'category_id': cat_id_learn[i],
# 'category_name': cat_name_learn[i],
# 'external_image_file': stimuli_learn_path[i],
# 'new_old_labels_recog': -1})
nwbfile.add_trial(start_time=events_learn_stim_on.iloc[i][0],
stop_time=events_learn_delay2_off.iloc[i][0],
stim_on=events_learn_stim_on.iloc[i][0],
stim_off=events_learn_stim_off.iloc[i][0],
delay1_off=events_learn_delay1_off.iloc[i][0],
delay2_off=events_learn_delay2_off.iloc[i][0],
stim_phase='learn',
category_id=cat_id_learn[i],
category_name=cat_name_learn[i],
external_image_file=stimuli_learn_path[i],
new_old_labels_recog='NA')
for i in range(range_recog):
# nwbfile.create_epoch(start_time=events_recog_stim_on.iloc[i][0],
# stop_time=events_recog_stim_off.iloc[i][0],
# timeseries=[event_ts, experiment_ids],
# tags='stimulus_recog',
# description='recognition phase stimulus')
nwbfile.add_trial(start_time=events_recog_stim_on.iloc[i][0],
stop_time=events_recog_delay2_off.iloc[i][0],
stim_on=events_recog_stim_on.iloc[i][0],
stim_off=events_recog_stim_off.iloc[i][0],
delay1_off=events_recog_delay1_off.iloc[i][0],
delay2_off=events_recog_delay2_off.iloc[i][0],
stim_phase='recog',
category_id=cat_id_recog[i],
category_name=cat_name_recog[i],
external_image_file=stimuli_recog_path[i],
new_old_labels_recog=new_old_recog[i])
# Add the waveform clustering and the spike data.
# Create necessary processing modules for different kinds of waveform data
clustering_processing_module = ProcessingModule(
'Spikes', 'The spike data contained')
clusterWaveform_learn_processing_module = ProcessingModule(
'MeanWaveforms_learn',
'The mean waveforms for the clustered raw signal for learning phase')
clusterWaveform_recog_processing_module = ProcessingModule(
'MeanWaveforms_recog',
'The mean waveforms for the clustered raw signal for recognition phase'
)
IsolDist_processing_module = ProcessingModule('IsoDist', 'The IsolDist')
SNR_processing_module = ProcessingModule('SNR', 'SNR (signal-to-noise)')
# Get the unique channel id that we will be iterate over
channel_ids = np.unique([cell_id[0] for cell_id in cell_ids])
# Interate the channel list
for channel_id in channel_ids:
cell_name = 'A' + str(channel_id) + '_cells.mat'
file_path = os.path.join('RecogMemory_MTL_release_v2', 'Data',
'sorted', session['session'], task_descr,
cell_name)
try:
cell_mat = loadmat(file_path)
except FileNotFoundError:
print("File not found")
continue
spikes = cell_mat['spikes']
meanWaveform_recog = cell_mat['meanWaveform_recog']
meanWaveform_learn = cell_mat['meanWaveform_learn']
IsolDist_SNR = cell_mat['IsolDist_SNR']
spike_id = np.asarray([spike[0] for spike in spikes])
spike_cluster_id = np.asarray([spike[1] for spike in spikes])
spike_timestamps = np.asarray([spike[2] / 1000000 for spike in spikes])
clustering = Clustering(description='Spikes of the channel detected',
num=spike_id,
peak_over_rms=np.asarray([0]),
times=spike_timestamps,
name='channel' + str(channel_id))
clustering_processing_module.add_data_interface(clustering)
for i in range(len(meanWaveform_learn[0][0][0][0])):
waveform_mean_learn = ClusterWaveforms(
clustering_interface=clustering,
waveform_filtering='NA',
waveform_sd=np.asarray([[0]]),
waveform_mean=np.asarray([meanWaveform_learn[0][0][1][i]]),
name='waveform_learn_cluster_id_' +
str(meanWaveform_learn[0][0][0][0][i]))
try:
clusterWaveform_learn_processing_module.add_data_interface(
waveform_mean_learn)
except ValueError as e:
print(
'Catch an error in adding waveform interface to the recog processing module:'
+ str(e))
continue
# Adding mean waveform recognition into the processing module
for i in range(len(meanWaveform_recog[0][0][0][0])):
waveform_mean_recog = ClusterWaveforms(
clustering_interface=clustering,
waveform_filtering='NA',
waveform_sd=np.asarray([[0]]),
waveform_mean=np.asarray([meanWaveform_recog[0][0][1][i]]),
name='waveform_recog_cluster_id_' +
str(meanWaveform_recog[0][0][0][0][i]))
try:
clusterWaveform_recog_processing_module.add_data_interface(
waveform_mean_recog)
except ValueError as e:
print(
'Catch an error in adding waveform interface to the recog processing module:'
+ str(e))
continue
# Adding IsolDist_SNR data into the processing module
# Here I use feature extraction to store the IsolDist_SNR data because
# they are extracted from the original signals.
# print(IsolDist_SNR[0][0][0])
for i in range(len(IsolDist_SNR[0][0][1][0])):
isoldist_data_interface = TimeSeries(
data=[IsolDist_SNR[0][0][1][0][i]],
unit='NA',
timestamps=[0],
name='IsolDist_' + str(IsolDist_SNR[0][0][0][0][i]))
try:
IsolDist_processing_module.add_data_interface(
isoldist_data_interface)
except ValueError as e:
print(
'Catch an error in adding IsolDist to the processing module:'
+ str(e))
continue
SNR_data_interface = TimeSeries(unit='NA',
description='The SNR data',
data=[IsolDist_SNR[0][0][2][0][i]],
timestamps=[0],
name='SNR_' +
str(IsolDist_SNR[0][0][0][0][i]))
try:
SNR_processing_module.add_data_interface(SNR_data_interface)
except ValueError as e:
print(
'Catch an error in adding SNR to the processing module:' +
str(e))
continue
nwbfile.add_processing_module(clustering_processing_module)
nwbfile.add_processing_module(clusterWaveform_learn_processing_module)
nwbfile.add_processing_module(clusterWaveform_recog_processing_module)
nwbfile.add_processing_module(IsolDist_processing_module)
nwbfile.add_processing_module(SNR_processing_module)
return nwbfile
def high_gamma_estimation(block_path, bands_vals, new_file=''):
"""
Takes preprocessed LFP data and calculates High-Gamma power from the
averaged power of standard Hilbert transform on 70~150 Hz bands.
Parameters
----------
block_path : str
subject file path
bands_vals : [2,nBands] numpy array with Gaussian filter parameters, where:
bands_vals[0,:] = filter centers [Hz]
bands_vals[1,:] = filter sigmas [Hz]
new_file : str
if this argument is of form 'path/to/new_file.nwb', High Gamma power
will be saved in a new file. If it is an empty string, '', High Gamma
power will be saved in the current NWB file.
Returns
-------
Saves High Gamma power (TimeSeries) in the current or new NWB file.
Only if container for this data do not exist in the file.
"""
# Get filter parameters
band_param_0 = bands_vals[0, :]
band_param_1 = bands_vals[1, :]
with NWBHDF5IO(block_path, 'r+', load_namespaces=True) as io:
nwb = io.read()
lfp = nwb.processing['ecephys'].data_interfaces[
'LFP'].electrical_series['preprocessed']
rate = lfp.rate
nBands = len(band_param_0)
nSamples = lfp.data.shape[0]
nChannels = lfp.data.shape[1]
Xp = np.zeros(
(nBands, nChannels, nSamples)) #power (nBands,nChannels,nSamples)
# Apply Hilbert transform ----------------------------------------------
print('Running High Gamma estimation...')
start = time.time()
for ch in np.arange(nChannels):
Xch = lfp.data[:,
ch] * 1e6 # 1e6 scaling helps with numerical accuracy
Xch = Xch.reshape(1, -1)
Xch = Xch.astype('float32') # signal (nChannels,nSamples)
X_fft_h = None
for ii, (bp0, bp1) in enumerate(zip(band_param_0, band_param_1)):
kernel = gaussian(Xch, rate, bp0, bp1)
X_analytic, X_fft_h = hilbert_transform(Xch,
rate,
kernel,
phase=None,
X_fft_h=X_fft_h)
Xp[ii, ch, :] = abs(X_analytic).astype('float32')
print(
'High Gamma estimation finished in {} seconds'.format(time.time() -
start))
# data: (ndarray) dims: num_times * num_channels * num_bands
Xp = np.swapaxes(Xp, 0, 2)
HG = np.mean(Xp, 2) #average of high gamma bands
# Storage of High Gamma on NWB file -----------------------------
if new_file == '': #on current file
#make electrodes table
nElecs = HG.shape[1]
elecs_region = nwb.electrodes.create_region(
name='electrodes',
region=np.arange(nElecs).tolist(),
description='all electrodes')
hg = ElectricalSeries(name='high_gamma',
data=HG,
electrodes=elecs_region,
rate=rate,
description='')
ecephys_module = nwb.processing['ecephys']
ecephys_module.add_data_interface(hg)
io.write(nwb)
print('High Gamma power saved in ' + block_path)
else: #on new file
with NWBHDF5IO(new_file, 'r+', load_namespaces=True) as io_new:
nwb_new = io_new.read()
#make electrodes table
nElecs = HG.shape[1]
elecs_region = nwb_new.electrodes.create_region(
name='electrodes',
region=np.arange(nElecs).tolist(),
description='all electrodes')
hg = ElectricalSeries(name='high_gamma',
data=HG,
electrodes=elecs_region,
rate=rate,
description='')
try: # if ecephys module already exists
ecephys_module = nwb_new.processing['ecephys']
except: # creates ecephys ProcessingModule
ecephys_module = ProcessingModule(
name='ecephys',
description='Extracellular electrophysiology data.')
nwb_new.add_processing_module(ecephys_module)
ecephys_module.add_data_interface(hg)
io_new.write(nwb_new)
print('High Gamma power saved in ' + new_file)
def preprocess_raw_data(block_path, config):
"""
Takes raw data and runs:
1) CAR
2) notch filters
3) Downsampling
Parameters
----------
block_path : str
subject file path
config : dictionary
'CAR' - Number of channels to use in CAR (default=16)
'Notch' - Main frequency (Hz) for notch filters (default=60)
'Downsample' - Downsampling frequency (Hz, default= 400)
Returns
-------
Saves preprocessed signals (LFP) in the current NWB file.
Only if containers for these data do not exist in the current file.
"""
subj_path, block_name = os.path.split(block_path)
block_name = os.path.splitext(block_path)[0]
start = time.time()
with NWBHDF5IO(block_path, 'r+', load_namespaces=True) as io:
nwb = io.read()
# Storage of processed signals on NWB file -----------------------------
try: # if ecephys module already exists
ecephys_module = nwb.processing['ecephys']
except: # creates ecephys ProcessingModule
ecephys_module = ProcessingModule(
name='ecephys',
description='Extracellular electrophysiology data.')
# Add module to NWB file
nwb.add_processing_module(ecephys_module)
print('Created ecephys')
# LFP: Downsampled and power line signal removed -----------------------
if 'LFP' in nwb.processing[
'ecephys'].data_interfaces: # if LFP already exists
lfp = nwb.processing['ecephys'].data_interfaces['LFP']
lfp_ts = nwb.processing['ecephys'].data_interfaces[
'LFP'].electrical_series['preprocessed']
else: # creates LFP data interface container
lfp = LFP()
# Data source
lis = list(nwb.acquisition.keys())
for i in lis: # Check if there is ElectricalSeries in acquisition group
if type(nwb.acquisition[i]).__name__ == 'ElectricalSeries':
source = nwb.acquisition[i]
nChannels = source.data.shape[1]
# Downsampling
extraBins0 = 0
fs = source.rate
if config['Downsample'] is not None:
print("Downsampling signals to " + str(config['Downsample']) +
" Hz.")
print("Please wait, this might take around 30 minutes.")
start = time.time()
#zeros to pad to make signal lenght a power of 2
nBins = source.data.shape[0]
extraBins0 = 2**(np.ceil(np.log2(nBins)).astype('int')) - nBins
extraZeros = np.zeros(extraBins0)
rate = config['Downsample']
#One channel at a time, to improve memory usage for long signals
for ch in np.arange(nChannels):
#1e6 scaling helps with numerical accuracy
Xch = source.data[:, ch] * 1e6
#Make lenght a power of 2, improves performance
Xch = np.append(Xch, extraZeros)
Xch = resample(Xch, rate, fs)
if ch == 0:
X = Xch.reshape(1, -1)
else:
X = np.append(X, Xch.reshape(1, -1), axis=0)
print(
'Downsampling finished in {} seconds'.format(time.time() -
start))
else: # No downsample
rate = fs
X = source.data[:, :].T * 1e6
# Subtract CAR
if config['CAR'] is not None:
print(
"Computing and subtracting Common Average Reference in " +
str(config['CAR']) + " channel blocks.")
start = time.time()
X = subtract_CAR(X, b_size=config['CAR'])
print('CAR subtract time for {}: {} seconds'.format(
block_name,
time.time() - start))
# Apply Notch filters
if config['Notch'] is not None:
print("Applying Notch filtering of " + str(config['Notch']) +
" Hz")
#zeros to pad to make signal lenght a power of 2
nBins = X.shape[1]
extraBins1 = 2**(np.ceil(np.log2(nBins)).astype('int')) - nBins
extraZeros = np.zeros(extraBins1)
start = time.time()
for ch in np.arange(nChannels):
Xch = np.append(X[ch, :], extraZeros).reshape(1, -1)
Xch = linenoise_notch(Xch,
rate,
notch_freq=config['Notch'])
if ch == 0:
X2 = Xch.reshape(1, -1)
else:
X2 = np.append(X2, Xch.reshape(1, -1), axis=0)
print('Notch filter time for {}: {} seconds'.format(
block_name,
time.time() - start))
X = np.copy(X2)
del X2
#Remove excess bins (because of zero padding on previous steps)
excessBins = int(np.ceil(extraBins0 * rate / fs) + extraBins1)
X = X[:, 0:-excessBins]
X = X.astype('float32') # signal (nChannels,nSamples)
X /= 1e6 # Scales signals back to Volt
# Add preprocessed downsampled signals as an electrical_series
if config['CAR'] is None:
car = 'None'
else:
car = str(config['CAR'])
if config['Notch'] is None:
notch = 'None'
else:
notch = str(config['Notch'])
if config['Downsample'] is None:
downs = 'No'
else:
downs = 'Yes'
config_comment = 'CAR:' + car + ', Notch:' + notch + ', Downsampled:' + downs
lfp_ts = lfp.create_electrical_series(name='preprocessed',
data=X.T,
electrodes=source.electrodes,
rate=rate,
description='',
comments=config_comment)
ecephys_module.add_data_interface(lfp)
# Write LFP to NWB file
io.write(nwb)
print('LFP saved in ' + block_path)
def add_cell_specimen_table(nwbfile: NWBFile,
cell_specimen_table: pd.DataFrame):
"""
This function takes the cell specimen table and writes the ROIs
contained within. It writes these to a new NWB imaging plane
based off the previously supplied metadata
Parameters
----------
nwbfile: NWBFile
this is the in memory NWBFile currently being written to which ROI data
is added
cell_specimen_table: pd.DataFrame
this is the DataFrame containing the cells segmented from a ophys
experiment, stored in json file and loaded.
example: /home/nicholasc/projects/allensdk/allensdk/test/
brain_observatory/behavior/cell_specimen_table_789359614.json
Returns
-------
nwbfile: NWBFile
The altered in memory NWBFile object that now has a specimen table
"""
cell_roi_table = cell_specimen_table.reset_index().set_index('cell_roi_id')
# Device:
device_name = nwbfile.lab_meta_data['metadata'].rig_name
nwbfile.create_device(device_name,
"Allen Brain Observatory")
device = nwbfile.get_device(device_name)
# Location:
location_description = "Area: {}, Depth: {} um".format(
nwbfile.lab_meta_data['metadata'].targeted_structure,
nwbfile.lab_meta_data['metadata'].imaging_depth)
# FOV:
fov_width = nwbfile.lab_meta_data['metadata'].field_of_view_width
fov_height = nwbfile.lab_meta_data['metadata'].field_of_view_height
imaging_plane_description = "{} field of view in {} at depth {} um".format(
(fov_width, fov_height),
nwbfile.lab_meta_data['metadata'].targeted_structure,
nwbfile.lab_meta_data['metadata'].imaging_depth)
# Optical Channel:
optical_channel = OpticalChannel(
name='channel_1',
description='2P Optical Channel',
emission_lambda=nwbfile.lab_meta_data['metadata'].emission_lambda)
# Imaging Plane:
imaging_plane = nwbfile.create_imaging_plane(
name='imaging_plane_1',
optical_channel=optical_channel,
description=imaging_plane_description,
device=device,
excitation_lambda=nwbfile.lab_meta_data['metadata'].excitation_lambda,
imaging_rate=nwbfile.lab_meta_data['metadata'].ophys_frame_rate,
indicator=nwbfile.lab_meta_data['metadata'].indicator,
location=location_description,
manifold=[], # Should this be passed in for future support?
conversion=1.0,
unit='unknown', # Should this be passed in for future support?
reference_frame='unknown') # Should this be passed in for future support?
# Image Segmentation:
image_segmentation = ImageSegmentation(name="image_segmentation")
if 'two_photon_imaging' not in nwbfile.modules:
two_photon_imaging_module = ProcessingModule('two_photon_imaging', '2P processing module')
nwbfile.add_processing_module(two_photon_imaging_module)
else:
two_photon_imaging_module = nwbfile.modules['two_photon_imaging']
two_photon_imaging_module.add_data_interface(image_segmentation)
# Plane Segmentation:
plane_segmentation = image_segmentation.create_plane_segmentation(
name='cell_specimen_table',
description="Segmented rois",
imaging_plane=imaging_plane)
for col_name in cell_roi_table.columns:
# the columns 'image_mask', 'pixel_mask', and 'voxel_mask' are already defined
# in the nwb.ophys::PlaneSegmentation Object
if col_name not in ['id', 'mask_matrix', 'image_mask', 'pixel_mask', 'voxel_mask']:
# This builds the columns with name of column and description of column
# both equal to the column name in the cell_roi_table
plane_segmentation.add_column(col_name,
CELL_SPECIMEN_COL_DESCRIPTIONS.get(col_name,
"No Description Available"))
# go through each roi and add it to the plan segmentation object
for cell_roi_id, row in cell_roi_table.iterrows():
sub_mask = np.array(row.pop('image_mask'))
curr_roi = roi.create_roi_mask(fov_width, fov_height, [(fov_width - 1), 0, (fov_height - 1), 0],
roi_mask=sub_mask)
mask = curr_roi.get_mask_plane()
csid = row.pop('cell_specimen_id')
row['cell_specimen_id'] = -1 if csid is None else csid
row['id'] = cell_roi_id
plane_segmentation.add_roi(image_mask=mask, **row.to_dict())
return nwbfile
def preprocess_raw_data(block_path, config):
"""
Takes raw data and runs:
1) CAR
2) notch filters
3) Downsampling
Parameters
----------
block_path : str
subject file path
config : dictionary
'referencing' - tuple specifying electrode referencing (type, options)
('CAR', N_channels_per_group)
('CMR', N_channels_per_group)
('bipolar', INCLUDE_OBLIQUE_NBHD)
'Notch' - Main frequency (Hz) for notch filters (default=60)
'Downsample' - Downsampling frequency (Hz, default= 400)
Returns
-------
Saves preprocessed signals (LFP) in the current NWB file.
Only if containers for these data do not exist in the current file.
"""
subj_path, block_name = os.path.split(block_path)
block_name = os.path.splitext(block_path)[0]
start = time.time()
with NWBHDF5IO(block_path, 'r+', load_namespaces=True) as io:
nwb = io.read()
# Storage of processed signals on NWB file ----------------------------
if 'ecephys' in nwb.processing:
ecephys_module = nwb.processing['ecephys']
else: # creates ecephys ProcessingModule
ecephys_module = ProcessingModule(
name='ecephys',
description='Extracellular electrophysiology data.')
# Add module to NWB file
nwb.add_processing_module(ecephys_module)
print('Created ecephys')
# LFP: Downsampled and power line signal removed ----------------------
if 'LFP' in nwb.processing['ecephys'].data_interfaces:
######
# What's the point of this? Nothing is done with these vars...
lfp = nwb.processing['ecephys'].data_interfaces['LFP']
lfp_ts = nwb.processing['ecephys'].data_interfaces[
'LFP'].electrical_series['preprocessed']
######
else: # creates LFP data interface container
lfp = LFP()
# Data source
source_list = [
acq for acq in nwb.acquisition.values()
if type(acq) == ElectricalSeries
]
assert len(source_list) == 1, (
'Not precisely one ElectricalSeries in acquisition!')
source = source_list[0]
nChannels = source.data.shape[1]
# Downsampling
if config['Downsample'] is not None:
print("Downsampling signals to " + str(config['Downsample']) +
" Hz.")
print("Please wait, this might take around 30 minutes.")
start = time.time()
# zeros to pad to make signal length a power of 2
nBins = source.data.shape[0]
extraBins0 = 2**(np.ceil(np.log2(nBins)).astype('int')) - nBins
extraZeros = np.zeros(extraBins0)
rate = config['Downsample']
# malloc
T = int(np.ceil((nBins + extraBins0) * rate / source.rate))
X = np.zeros((source.data.shape[1], T))
# One channel at a time, to improve memory usage for long signals
for ch in np.arange(nChannels):
# 1e6 scaling helps with numerical accuracy
Xch = source.data[:, ch] * 1e6
# Make length a power of 2, improves performance
Xch = np.append(Xch, extraZeros)
X[ch, :] = resample(Xch, rate, source.rate)
print(
'Downsampling finished in {} seconds'.format(time.time() -
start))
else: # No downsample
extraBins0 = 0
rate = source.rate
X = source.data[()].T * 1e6
# re-reference the (scaled by 1e6!) data
electrodes = source.electrodes
if config['referencing'] is not None:
if config['referencing'][0] == 'CAR':
print(
"Computing and subtracting Common Average Reference in "
+ str(config['referencing'][1]) + " channel blocks.")
start = time.time()
X = subtract_CAR(X, b_size=config['referencing'][1])
print('CAR subtract time for {}: {} seconds'.format(
block_name,
time.time() - start))
elif config['referencing'][0] == 'bipolar':
X, bipolarTable, electrodes = get_bipolar_referenced_electrodes(
X, electrodes, rate, grid_step=1)
# add data interface for the metadata for saving
ecephys_module.add_data_interface(bipolarTable)
print('bipolarElectrodes stored for saving in ' +
block_path)
else:
print('UNRECOGNIZED REFERENCING SCHEME; ', end='')
print('SKIPPING REFERENCING!')
# Apply Notch filters
if config['Notch'] is not None:
print("Applying notch filtering of " + str(config['Notch']) +
" Hz")
# zeros to pad to make signal lenght a power of 2
nBins = X.shape[1]
extraBins1 = 2**(np.ceil(np.log2(nBins)).astype('int')) - nBins
extraZeros = np.zeros(extraBins1)
start = time.time()
for ch in np.arange(nChannels):
Xch = np.append(X[ch, :], extraZeros).reshape(1, -1)
Xch = linenoise_notch(Xch,
rate,
notch_freq=config['Notch'])
if ch == 0:
X2 = Xch.reshape(1, -1)
else:
X2 = np.append(X2, Xch.reshape(1, -1), axis=0)
print('Notch filter time for {}: {} seconds'.format(
block_name,
time.time() - start))
X = np.copy(X2)
del X2
else:
extraBins1 = 0
# Remove excess bins (because of zero padding on previous steps)
excessBins = int(
np.ceil(extraBins0 * rate / source.rate) + extraBins1)
X = X[:, 0:-excessBins]
X = X.astype('float32') # signal (nChannels,nSamples)
X /= 1e6 # Scales signals back to volts
# Add preprocessed downsampled signals as an electrical_series
referencing = 'None' if config['referencing'] is None else config[
'referencing'][0]
notch = 'None' if config['Notch'] is None else str(config['Notch'])
downs = 'No' if config['Downsample'] is None else 'Yes'
config_comment = ('referencing:' + referencing + ',Notch:' +
notch + ', Downsampled:' + downs)
# create an electrical series for the LFP and store it in lfp
lfp_ts = lfp.create_electrical_series(name='preprocessed',
data=X.T,
electrodes=electrodes,
rate=rate,
description='',
comments=config_comment)
ecephys_module.add_data_interface(lfp)
# Write LFP to NWB file
io.write(nwb)
print('LFP saved in ' + block_path)
def nwb_copy_file(old_file, new_file, cp_objs={}):
"""
Copy fields defined in 'obj', from existing NWB file to new NWB file.
Parameters
----------
old_file : str, path
String such as '/path/to/old_file.nwb'.
new_file : str, path
String such as '/path/to/new_file.nwb'.
cp_objs : dict
Name:Value pairs (Group:Children) listing the groups and respective
children from the current NWB file to be copied. Children can be:
- Boolean, indicating an attribute (e.g. for institution, lab)
- List of strings, containing several children names
Example:
{'institution':True,
'lab':True,
'acquisition':['microphone'],
'ecephys':['LFP','DecompositionSeries']}
"""
manager = get_manager()
# Open original signal file
with NWBHDF5IO(old_file, 'r', manager=manager,
load_namespaces=True) as io1:
nwb_old = io1.read()
# Creates new file
nwb_new = NWBFile(session_description=str(nwb_old.session_description),
identifier='',
session_start_time=datetime.now(tzlocal()))
with NWBHDF5IO(new_file, mode='w', manager=manager,
load_namespaces=False) as io2:
# Institution name ------------------------------------------------
if 'institution' in cp_objs:
nwb_new.institution = str(nwb_old.institution)
# Lab name --------------------------------------------------------
if 'lab' in cp_objs:
nwb_new.lab = str(nwb_old.lab)
# Session id ------------------------------------------------------
if 'session' in cp_objs:
nwb_new.session_id = nwb_old.session_id
# Devices ---------------------------------------------------------
if 'devices' in cp_objs:
for aux in list(nwb_old.devices.keys()):
dev = Device(nwb_old.devices[aux].name)
nwb_new.add_device(dev)
# Electrode groups ------------------------------------------------
if 'electrode_groups' in cp_objs:
for aux in list(nwb_old.electrode_groups.keys()):
nwb_new.create_electrode_group(
name=str(nwb_old.electrode_groups[aux].name),
description=str(nwb_old.electrode_groups[
aux].description),
location=str(nwb_old.electrode_groups[aux].location),
device=nwb_new.get_device(
nwb_old.electrode_groups[aux].device.name)
)
# Electrodes ------------------------------------------------------
if 'electrodes' in cp_objs:
nElec = len(nwb_old.electrodes['x'].data[:])
for aux in np.arange(nElec):
nwb_new.add_electrode(
x=nwb_old.electrodes['x'][aux],
y=nwb_old.electrodes['y'][aux],
z=nwb_old.electrodes['z'][aux],
imp=nwb_old.electrodes['imp'][aux],
location=str(nwb_old.electrodes['location'][aux]),
filtering=str(nwb_old.electrodes['filtering'][aux]),
group=nwb_new.get_electrode_group(
nwb_old.electrodes['group'][aux].name),
group_name=str(nwb_old.electrodes['group_name'][aux])
)
# if there are custom variables
new_vars = list(nwb_old.electrodes.colnames)
default_vars = ['x', 'y', 'z', 'imp', 'location', 'filtering',
'group', 'group_name']
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
if var == 'label':
var_data = [str(elem) for elem in nwb_old.electrodes[
var].data[:]]
else:
var_data = np.array(nwb_old.electrodes[var].data[:])
nwb_new.add_electrode_column(name=str(var),
description=
str(nwb_old.electrodes[
var].description),
data=var_data)
# Epochs ----------------------------------------------------------
if 'epochs' in cp_objs:
nEpochs = len(nwb_old.epochs['start_time'].data[:])
for i in np.arange(nEpochs):
nwb_new.add_epoch(
start_time=nwb_old.epochs['start_time'].data[i],
stop_time=nwb_old.epochs['stop_time'].data[i])
# if there are custom variables
new_vars = list(nwb_old.epochs.colnames)
default_vars = ['start_time', 'stop_time', 'tags',
'timeseries']
[new_vars.remove(var) for var in default_vars if
var in new_vars]
for var in new_vars:
nwb_new.add_epoch_column(name=var,
description=nwb_old.epochs[
var].description,
data=nwb_old.epochs[var].data[:])
# Invalid times ---------------------------------------------------
if 'invalid_times' in cp_objs:
nInvalid = len(nwb_old.invalid_times['start_time'][:])
for aux in np.arange(nInvalid):
nwb_new.add_invalid_time_interval(
start_time=nwb_old.invalid_times['start_time'][aux],
stop_time=nwb_old.invalid_times['stop_time'][aux])
# Trials ----------------------------------------------------------
if 'trials' in cp_objs:
nTrials = len(nwb_old.trials['start_time'])
for aux in np.arange(nTrials):
nwb_new.add_trial(
start_time=nwb_old.trials['start_time'][aux],
stop_time=nwb_old.trials['stop_time'][aux])
# if there are custom variables
new_vars = list(nwb_old.trials.colnames)
default_vars = ['start_time', 'stop_time']
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
nwb_new.add_trial_column(name=var,
description=nwb_old.trials[
var].description,
data=nwb_old.trials[var].data[:])
# Intervals -------------------------------------------------------
if 'intervals' in cp_objs:
all_objs_names = list(nwb_old.intervals.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.intervals[obj_name]
# create and add TimeIntervals
obj = TimeIntervals(name=obj_old.name,
description=obj_old.description)
nInt = len(obj_old['start_time'])
for ind in np.arange(nInt):
obj.add_interval(start_time=obj_old['start_time'][ind],
stop_time=obj_old['stop_time'][ind])
# Add to file
nwb_new.add_time_intervals(obj)
# Stimulus --------------------------------------------------------
if 'stimulus' in cp_objs:
all_objs_names = list(nwb_old.stimulus.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.stimulus[obj_name]
obj = TimeSeries(name=obj_old.name,
description=obj_old.description,
data=obj_old.data[:],
rate=obj_old.rate,
resolution=obj_old.resolution,
conversion=obj_old.conversion,
starting_time=obj_old.starting_time,
unit=obj_old.unit)
nwb_new.add_stimulus(obj)
# Processing modules ----------------------------------------------
if 'ecephys' in cp_objs:
if cp_objs['ecephys'] is True:
interfaces = nwb_old.processing[
'ecephys'].data_interfaces.keys()
else: # list of items
interfaces = [
nwb_old.processing['ecephys'].data_interfaces[key]
for key in cp_objs['ecephys']
]
# Add ecephys module to NWB file
ecephys_module = ProcessingModule(
name='ecephys',
description='Extracellular electrophysiology data.'
)
nwb_new.add_processing_module(ecephys_module)
for interface_old in interfaces:
obj = copy_obj(interface_old, nwb_old, nwb_new)
if obj is not None:
ecephys_module.add_data_interface(obj)
# Acquisition -----------------------------------------------------
if 'acquisition' in cp_objs:
if cp_objs['acquisition'] is True:
all_acq_names = list(nwb_old.acquisition.keys())
else: # list of items
all_acq_names = cp_objs['acquisition']
for acq_name in all_acq_names:
obj_old = nwb_old.acquisition[acq_name]
obj = copy_obj(obj_old, nwb_old, nwb_new)
if obj is not None:
nwb_new.add_acquisition(obj)
# Subject ---------------------------------------------------------
if 'subject' in cp_objs:
try:
cortical_surfaces = CorticalSurfaces()
surfaces = nwb_old.subject.cortical_surfaces.surfaces
for sfc in list(surfaces.keys()):
cortical_surfaces.create_surface(
name=surfaces[sfc].name,
faces=surfaces[sfc].faces,
vertices=surfaces[sfc].vertices)
nwb_new.subject = ECoGSubject(
cortical_surfaces=cortical_surfaces,
subject_id=nwb_old.subject.subject_id,
age=nwb_old.subject.age,
description=nwb_old.subject.description,
genotype=nwb_old.subject.genotype,
sex=nwb_old.subject.sex,
species=nwb_old.subject.species,
weight=nwb_old.subject.weight,
date_of_birth=nwb_old.subject.date_of_birth)
except:
nwb_new.subject = Subject(age=nwb_old.subject.age,
description=nwb_old.subject.description,
genotype=nwb_old.subject.genotype,
sex=nwb_old.subject.sex,
species=nwb_old.subject.species,
subject_id=nwb_old.subject.subject_id,
weight=nwb_old.subject.weight,
date_of_birth=nwb_old.subject.date_of_birth)
# Write new file with copied fields
io2.write(nwb_new, link_data=False)
def add_cell_specimen_table(nwbfile: NWBFile,
cell_specimen_table: pd.DataFrame,
session_metadata: dict):
"""
This function takes the cell specimen table and writes the ROIs
contained within. It writes these to a new NWB imaging plane
based off the previously supplied metadata
Parameters
----------
nwbfile: NWBFile
this is the in memory NWBFile currently being written to which ROI data
is added
cell_specimen_table: pd.DataFrame
this is the DataFrame containing the cells segmented from a ophys
experiment, stored in json file and loaded.
example: /home/nicholasc/projects/allensdk/allensdk/test/
brain_observatory/behavior/cell_specimen_table_789359614.json
session_metadata: dict
Dictionary containing cell_specimen_table related metadata. Should
include at minimum the following fields:
"emission_lambda", "excitation_lambda", "indicator",
"targeted_structure", and ophys_frame_rate"
Returns
-------
nwbfile: NWBFile
The altered in memory NWBFile object that now has a specimen table
"""
cell_specimen_metadata = NwbOphysMetadataSchema().load(
session_metadata, unknown=marshmallow.EXCLUDE)
cell_roi_table = cell_specimen_table.reset_index().set_index('cell_roi_id')
# Device:
device_name: str = nwbfile.lab_meta_data['metadata'].equipment_name
if device_name.startswith("MESO"):
device_config = {
"name": device_name,
"description": "Allen Brain Observatory - Mesoscope 2P Rig"
}
else:
device_config = {
"name": device_name,
"description": "Allen Brain Observatory - Scientifica 2P Rig",
"manufacturer": "Scientifica"
}
nwbfile.create_device(**device_config)
device = nwbfile.get_device(device_name)
# FOV:
fov_width = nwbfile.lab_meta_data['metadata'].field_of_view_width
fov_height = nwbfile.lab_meta_data['metadata'].field_of_view_height
imaging_plane_description = "{} field of view in {} at depth {} um".format(
(fov_width, fov_height), cell_specimen_metadata['targeted_structure'],
nwbfile.lab_meta_data['metadata'].imaging_depth)
# Optical Channel:
optical_channel = OpticalChannel(
name='channel_1',
description='2P Optical Channel',
emission_lambda=cell_specimen_metadata['emission_lambda'])
# Imaging Plane:
imaging_plane = nwbfile.create_imaging_plane(
name='imaging_plane_1',
optical_channel=optical_channel,
description=imaging_plane_description,
device=device,
excitation_lambda=cell_specimen_metadata['excitation_lambda'],
imaging_rate=cell_specimen_metadata['ophys_frame_rate'],
indicator=cell_specimen_metadata['indicator'],
location=cell_specimen_metadata['targeted_structure'])
# Image Segmentation:
image_segmentation = ImageSegmentation(name="image_segmentation")
if 'ophys' not in nwbfile.processing:
ophys_module = ProcessingModule('ophys', 'Ophys processing module')
nwbfile.add_processing_module(ophys_module)
else:
ophys_module = nwbfile.processing['ophys']
ophys_module.add_data_interface(image_segmentation)
# Plane Segmentation:
plane_segmentation = image_segmentation.create_plane_segmentation(
name='cell_specimen_table',
description="Segmented rois",
imaging_plane=imaging_plane)
for col_name in cell_roi_table.columns:
# the columns 'roi_mask', 'pixel_mask', and 'voxel_mask' are
# already defined in the nwb.ophys::PlaneSegmentation Object
if col_name not in [
'id', 'mask_matrix', 'roi_mask', 'pixel_mask', 'voxel_mask'
]:
# This builds the columns with name of column and description
# of column both equal to the column name in the cell_roi_table
plane_segmentation.add_column(
col_name,
CELL_SPECIMEN_COL_DESCRIPTIONS.get(col_name,
"No Description Available"))
# go through each roi and add it to the plan segmentation object
for cell_roi_id, table_row in cell_roi_table.iterrows():
# NOTE: The 'roi_mask' in this cell_roi_table has already been
# processing by the function from
# allensdk.brain_observatory.behavior.session_apis.data_io.ophys_lims_api
# get_cell_specimen_table() method. As a result, the ROI is stored in
# an array that is the same shape as the FULL field of view of the
# experiment (e.g. 512 x 512).
mask = table_row.pop('roi_mask')
csid = table_row.pop('cell_specimen_id')
table_row['cell_specimen_id'] = -1 if csid is None else csid
table_row['id'] = cell_roi_id
plane_segmentation.add_roi(image_mask=mask, **table_row.to_dict())
return nwbfile
def psd_estimate(src_file, type):
"""
Estimates Power Spectral Density from signals.
Parameters
----------
src_file : str or path
Full path to the current NWB file.
type : str
ElectricalSeries source. 'raw' or 'preprocessed'.
"""
# Open file
with NWBHDF5IO(src_file, mode='r+', load_namespaces=True) as io:
nwb = io.read()
# Source ElectricalSeries
if type == 'raw':
# Check if there is ElectricalSeries in acquisition group
for i in list(nwb.acquisition.keys()):
if isinstance(nwb.acquisition[i], ElectricalSeries):
data_obj = nwb.acquisition[i]
elif type == 'preprocessed':
data_obj = nwb.processing['ecephys'].data_interfaces['LFP'].electrical_series['preprocessed']
nChannels = data_obj.data.shape[1]
nSamples = data_obj.data.shape[0]
fs = data_obj.rate
# Welch - window length as power of 2 and keeps dF~0.05 Hz
dF = .05 # Frequency bin size
win_len_welch = 2**(np.ceil(np.log2(fs / dF)).astype('int')) # dF = fs/nfft
# FFT - using a power of 2 number of samples improves performance
nfft = int(2**(np.floor(np.log2(nSamples)).astype('int')))
fx_lim = 200.
for ch in np.arange(nChannels): # Iterate over channels
trace = data_obj.data[:, ch]
fx_w, py_w = sgn.welch(trace, fs=fs, nperseg=win_len_welch)
fx_f, py_f = sgn.periodogram(trace, fs=fs, nfft=nfft)
# saves PSD up to 200 Hz
py_w = py_w[fx_w < fx_lim]
fx_w = fx_w[fx_w < fx_lim]
py_f = py_f[fx_f < fx_lim]
fx_f = fx_f[fx_f < fx_lim]
if ch == 0:
PY_welch = py_w.reshape(-1, 1)
PY_fft = py_f.reshape(-1, 1)
else:
PY_welch = np.append(PY_welch, py_w.reshape(-1, 1), axis=1)
PY_fft = np.append(PY_fft, py_f.reshape(-1, 1), axis=1)
# vElectrodes
elecs_region = nwb.electrodes.create_region(name='electrodes',
region=np.arange(nChannels).tolist(),
description='all electrodes')
# vPSD shape: ('frequency', 'channel')
spectrum_module_welch = Spectrum(name='Spectrum_welch_' + type,
frequencies=fx_w,
power=PY_welch,
source_timeseries=data_obj,
electrodes=elecs_region)
spectrum_module_fft = Spectrum(name='Spectrum_fft_' + type,
frequencies=fx_f,
power=PY_fft,
source_timeseries=data_obj,
electrodes=elecs_region)
# Processing module
try: # if ecephys module already exists
ecephys_module = nwb.processing['ecephys']
except: # creates ecephys ProcessingModule
ecephys_module = ProcessingModule(name='ecephys',
description='Extracellular electrophysiology data.')
# Add module to NWB file
nwb.add_processing_module(ecephys_module)
print('Created ecephys')
ecephys_module.add_data_interface(spectrum_module_welch)
ecephys_module.add_data_interface(spectrum_module_fft)
io.write(nwb)
print('Spectrum_welch_' + type + ' added to file.')
print('Spectrum_fft_' + type + ' added to file.')
def add_cell_specimen_table(nwbfile, cell_specimen_table):
cell_roi_table = cell_specimen_table.reset_index().set_index('cell_roi_id')
# Device:
device_name = nwbfile.lab_meta_data['metadata'].rig_name
nwbfile.create_device(device_name, "Allen Brain Observatory")
device = nwbfile.get_device(device_name)
# Location:
location_description = "Area: {}, Depth: {} um".format(
nwbfile.lab_meta_data['metadata'].targeted_structure,
nwbfile.lab_meta_data['metadata'].imaging_depth)
# FOV:
fov_width = nwbfile.lab_meta_data['metadata'].field_of_view_width
fov_height = nwbfile.lab_meta_data['metadata'].field_of_view_height
imaging_plane_description = "{} field of view in {} at depth {} um".format(
(fov_width, fov_height),
nwbfile.lab_meta_data['metadata'].targeted_structure,
nwbfile.lab_meta_data['metadata'].imaging_depth)
# Optical Channel:
optical_channel = OpticalChannel(
name='channel_1',
description='2P Optical Channel',
emission_lambda=nwbfile.lab_meta_data['metadata'].emission_lambda)
# Imaging Plane:
imaging_plane = nwbfile.create_imaging_plane(
name='imaging_plane_1',
optical_channel=optical_channel,
description=imaging_plane_description,
device=device,
excitation_lambda=nwbfile.lab_meta_data['metadata'].excitation_lambda,
imaging_rate=nwbfile.lab_meta_data['metadata'].ophys_frame_rate,
indicator=nwbfile.lab_meta_data['metadata'].indicator,
location=location_description,
manifold=[], # Should this be passed in for future support?
conversion=1.0,
unit='unknown', # Should this be passed in for future support?
reference_frame='unknown'
) # Should this be passed in for future support?
# Image Segmentation:
image_segmentation = ImageSegmentation(name="image_segmentation")
if 'two_photon_imaging' not in nwbfile.modules:
two_photon_imaging_module = ProcessingModule('two_photon_imaging',
'2P processing module')
nwbfile.add_processing_module(two_photon_imaging_module)
else:
two_photon_imaging_module = nwbfile.modules['two_photon_imaging']
two_photon_imaging_module.add_data_interface(image_segmentation)
# Plane Segmentation:
plane_segmentation = image_segmentation.create_plane_segmentation(
name='cell_specimen_table',
description="Segmented rois",
imaging_plane=imaging_plane)
for c in [
c for c in cell_roi_table.columns
if c not in ['id', 'mask_matrix']
]:
plane_segmentation.add_column(c, c)
for cell_roi_id, row in cell_roi_table.iterrows():
sub_mask = np.array(row.pop('image_mask'))
curr_roi = roi.create_roi_mask(fov_width,
fov_height, [(fov_width - 1), 0,
(fov_height - 1), 0],
roi_mask=sub_mask)
mask = curr_roi.get_mask_plane()
csid = row.pop('cell_specimen_id')
row['cell_specimen_id'] = -1 if csid is None else csid
row['id'] = cell_roi_id
plane_segmentation.add_roi(image_mask=mask, **row.to_dict())
return nwbfile
def preprocess_raw_data(block_path, config):
"""
Takes raw data and runs:
1) CAR
2) notch filters
3) Downsampling
Parameters
----------
block_path : str
subject file path
config : dictionary
'referencing' - tuple specifying electrode referencing (type, options)
('CAR', N_channels_per_group)
('CMR', N_channels_per_group)
('bipolar', INCLUDE_OBLIQUE_NBHD)
'Notch' - Main frequency (Hz) for notch filters (default=60)
'Downsample' - Downsampling frequency (Hz, default= 400)
Returns
-------
Saves preprocessed signals (LFP) in the current NWB file.
Only if containers for these data do not exist in the current file.
"""
subj_path, block_name = os.path.split(block_path)
block_name = os.path.splitext(block_path)[0]
start = time.time()
with NWBHDF5IO(block_path, 'r+', load_namespaces=True) as io:
nwb = io.read()
# Storage of processed signals on NWB file ----------------------------
if 'ecephys' in nwb.processing:
ecephys_module = nwb.processing['ecephys']
else: # creates ecephys ProcessingModule
ecephys_module = ProcessingModule(
name='ecephys',
description='Extracellular electrophysiology data.')
# Add module to NWB file
nwb.add_processing_module(ecephys_module)
print('Created ecephys')
# LFP: Downsampled and power line signal removed ----------------------
if 'LFP' in nwb.processing['ecephys'].data_interfaces:
warnings.warn(
'LFP data already exists in the nwb file. Skipping preprocessing.'
)
else: # creates LFP data interface container
lfp = LFP()
# Data source
source_list = [
acq for acq in nwb.acquisition.values()
if type(acq) == ElectricalSeries
]
assert len(source_list) == 1, (
'Not precisely one ElectricalSeries in acquisition!')
source = source_list[0]
nChannels = source.data.shape[1]
# Downsampling
if config['Downsample'] is not None:
print("Downsampling signals to " + str(config['Downsample']) +
" Hz.")
print("Please wait...")
start = time.time()
# Note: zero padding the signal to make the length
# a power of 2 won't help, since resample will further pad it
# (breaking the power of 2)
nBins = source.data.shape[0]
rate = config['Downsample']
# malloc
T = int(np.ceil(nBins * rate / source.rate))
X = np.zeros((source.data.shape[1], T))
# One channel at a time, to improve memory usage for long signals
for ch in np.arange(nChannels):
# 1e6 scaling helps with numerical accuracy
Xch = source.data[:, ch] * 1e6
X[ch, :] = resample(Xch, rate, source.rate)
print(
'Downsampling finished in {} seconds'.format(time.time() -
start))
else: # No downsample
rate = source.rate
X = source.data[()].T * 1e6
# re-reference the (scaled by 1e6!) data
electrodes = source.electrodes
if config['referencing'] is not None:
if config['referencing'][0] == 'CAR':
print(
"Computing and subtracting Common Average Reference in "
+ str(config['referencing'][1]) + " channel blocks.")
start = time.time()
X = subtract_CAR(X, b_size=config['referencing'][1])
print('CAR subtract time for {}: {} seconds'.format(
block_name,
time.time() - start))
elif config['referencing'][0] == 'bipolar':
X, bipolarTable, electrodes = get_bipolar_referenced_electrodes(
X, electrodes, rate, grid_step=1)
# add data interface for the metadata for saving
ecephys_module.add_data_interface(bipolarTable)
print('bipolarElectrodes stored for saving in ' +
block_path)
else:
print('UNRECOGNIZED REFERENCING SCHEME; ', end='')
print('SKIPPING REFERENCING!')
# Apply Notch filters
if config['Notch'] is not None:
print("Applying notch filtering of " + str(config['Notch']) +
" Hz")
# Note: zero padding the signal to make the length a power
# of 2 won't help, since notch filtering will further pad it
start = time.time()
for ch in np.arange(nChannels):
# NOTE: apply_linenoise_notch takes a signal that is
# (n_timePoints, n_channels). The documentation may be wrong
Xch = X[ch, :].reshape(-1, 1)
Xch = apply_linenoise_notch(Xch, rate)
X[ch, :] = Xch[:, 0]
print('Notch filter time for {}: {} seconds'.format(
block_name,
time.time() - start))
X = X.astype('float32') # signal (nChannels,nSamples)
X /= 1e6 # Scales signals back to volts
# Add preprocessed downsampled signals as an electrical_series
referencing = 'None' if config['referencing'] is None else config[
'referencing'][0]
notch = 'None' if config['Notch'] is None else str(config['Notch'])
downs = 'No' if config['Downsample'] is None else 'Yes'
config_comment = ('referencing:' + referencing + ', Notch:' +
notch + ', Downsampled:' + downs)
# create an electrical series for the LFP and store it in lfp
lfp.create_electrical_series(name='preprocessed',
data=X.T,
electrodes=electrodes,
rate=rate,
description='',
comments=config_comment)
ecephys_module.add_data_interface(lfp)
# Write LFP to NWB file
io.write(nwb)
print('LFP saved in ' + block_path)