def run_conversion(self, nwbfile: NWBFile, metadata: dict):
"""
Run conversion for this data interface.
Reads treadmill experiment behavioral data from csv files and adds it to nwbfile.
Parameters
----------
nwbfile : NWBFile
metadata : dict
"""
# Detect relevant files: trials summary, treadmill data and nose data
dir_behavior_treadmill = self.source_data['dir_behavior_treadmill']
trials_file = [
f for f in Path(dir_behavior_treadmill).glob('*_tr.csv')
if '~lock' not in f.name
][0]
treadmill_file = trials_file.name.split('_tr')[0] + '.csv'
nose_file = trials_file.name.split('_tr')[0] + '_mk.csv'
trials_file = os.path.join(dir_behavior_treadmill, trials_file)
treadmill_file = os.path.join(dir_behavior_treadmill, treadmill_file)
nose_file = os.path.join(dir_behavior_treadmill, nose_file)
# Add trials
if nwbfile.trials is not None:
print(
'Trials already exist in current nwb file. Treadmill behavior trials not added.'
)
else:
df_trials_summary = pd.read_csv(trials_file)
nwbfile.add_trial_column(name='fail', description='no description')
nwbfile.add_trial_column(name='reward_given',
description='no description')
nwbfile.add_trial_column(name='total_rewards',
description='no description')
nwbfile.add_trial_column(name='init_dur',
description='no description')
nwbfile.add_trial_column(name='light_dur',
description='no description')
nwbfile.add_trial_column(name='motor_dur',
description='no description')
nwbfile.add_trial_column(name='post_motor',
description='no description')
nwbfile.add_trial_column(name='speed',
description='no description')
nwbfile.add_trial_column(name='speed_mode',
description='no description')
nwbfile.add_trial_column(name='amplitude',
description='no description')
nwbfile.add_trial_column(name='period',
description='no description')
nwbfile.add_trial_column(name='deviation',
description='no description')
t_offset = df_trials_summary.loc[0]['Start Time']
for index, row in df_trials_summary.iterrows():
nwbfile.add_trial(
start_time=row['Start Time'] - t_offset,
stop_time=row['End Time'] - t_offset,
fail=row['Fail'],
reward_given=row['Reward Given'],
total_rewards=row['Total Rewards'],
init_dur=row['Init Dur'],
light_dur=row['Light Dur'],
motor_dur=row['Motor Dur'],
post_motor=row['Post Motor'],
speed=row['Speed'],
speed_mode=row['Speed Mode'],
amplitude=row['Amplitude'],
period=row['Period'],
deviation=row['+/- Deviation'],
)
# Treadmill continuous data
df_treadmill = pd.read_csv(treadmill_file, index_col=False)
# Nose position continuous data
df_nose = pd.read_csv(nose_file, index_col=False)
# All behavioral data
df_all = pd.concat([df_treadmill, df_nose], axis=1, sort=False)
meta_behavioral_ts = metadata['Behavior']
t_offset = df_treadmill.loc[0]['Time']
for meta in meta_behavioral_ts.values():
ts = TimeSeries(name=meta['name'],
data=df_all[meta['name']].to_numpy(),
timestamps=df_all['Time'].to_numpy() - t_offset,
description=meta['description'])
nwbfile.add_acquisition(ts)
def run_conversion(self,
nwbfile: NWBFile,
metadata: dict,
stub_test: bool = False,
external_mode: bool = True,
starting_times: Optional[list] = None,
chunk_data: bool = True,
module_name: Optional[str] = None,
module_description: Optional[str] = None):
"""
Convert the movie data files to ImageSeries and write them in the NWBFile.
Parameters
----------
nwbfile : NWBFile
metadata : dict
stub_test : bool, optional
If True, truncates the write operation for fast testing. The default is False.
external_mode : bool, optional
ImageSeries in NWBFiles may contain either explicit movie data or file paths to external movie files. If
True, this utilizes the more efficient method of merely encoding the file path linkage (recommended). For
data sharing, the video files must be contained in the same folder as the NWBFile. If the intention of this
NWBFile involves an upload to DANDI, the non-NWBFile types are not allowed so this flag would have to be
set to False. The default is True.
starting_times : list, optional
List of start times for each movie. If unspecified, assumes that the movies in the file_paths list are in
sequential order and are contiguous.
chunk_data : bool, optional
If True, uses a DataChunkIterator to read and write the movie, reducing overhead RAM usage at the cost of
reduced conversion speed (compared to loading video entirely into RAM as an array). This will also force to
True, even if manually set to False, whenever the video file size exceeds available system RAM by a factor
of 70 (from compression experiments). Based on experiements for a ~30 FPS system of ~400 x ~600 color
frames, the equivalent uncompressed RAM usage is around 2GB per minute of video. The default is True.
module_name: str, optional
Name of the processing module to add the ImageSeries object to. Default behavior is to add as acquisition.
module_description: str, optional
If the processing module specified by module_name does not exist, it will be created with this description.
The default description is the same as used by the conversion_tools.get_module function.
"""
file_paths = self.source_data['file_paths']
if stub_test:
count_max = 10
else:
count_max = np.inf
if starting_times is not None:
assert isinstance(starting_times, list) and all([isinstance(x, float) for x in starting_times]) \
and len(starting_times) == len(file_paths), \
"Argument 'starting_times' must be a list of floats in one-to-one correspondence with 'file_paths'!"
else:
starting_times = [0.]
for j, file in enumerate(file_paths):
timestamps = starting_times[j] + get_movie_timestamps(
movie_file=file)
if len(starting_times) != len(file_paths):
starting_times.append(timestamps[-1])
image_series_kwargs = dict(name=f"Video: {Path(file).stem}",
description="Video recorded by camera.",
unit="Frames")
if check_regular_timestamps(ts=timestamps):
fps = get_movie_fps(movie_file=file)
image_series_kwargs.update(starting_time=starting_times[j],
rate=fps)
else:
image_series_kwargs.update(
timestamps=H5DataIO(timestamps, compression="gzip"))
if external_mode:
image_series_kwargs.update(format="external",
external_file=[file])
else:
uncompressed_estimate = Path(file).stat().st_size * 70
available_memory = psutil.virtual_memory().available
if not chunk_data and uncompressed_estimate >= available_memory:
warn(
f"Not enough memory (estimated {round(uncompressed_estimate/1e9, 2)} GB) to load movie file as "
f"array ({round(available_memory/1e9, 2)} GB available)! Forcing chunk_data to True."
)
chunk_data = True
total_frames = len(timestamps)
frame_shape = get_frame_shape(movie_file=file)
maxshape = [total_frames]
maxshape.extend(frame_shape)
best_gzip_chunk = (1, frame_shape[0], frame_shape[1], 3)
tqdm_pos, tqdm_mininterval = (0, 10)
if chunk_data:
def data_generator(file, count_max):
cap = cv2.VideoCapture(str(file))
for _ in range(min(count_max, total_frames)):
success, frame = cap.read()
yield frame
cap.release()
mov = DataChunkIterator(
data=tqdm(
iterable=data_generator(file=file,
count_max=count_max),
desc=f"Copying movie data for {Path(file).name}",
position=tqdm_pos,
total=total_frames,
mininterval=tqdm_mininterval),
iter_axis=0, # nwb standard is time as zero axis
maxshape=tuple(maxshape))
image_series_kwargs.update(data=H5DataIO(
mov, compression="gzip", chunks=best_gzip_chunk))
else:
cap = cv2.VideoCapture(str(file))
mov = []
with tqdm(desc=f"Reading movie data for {Path(file).name}",
position=tqdm_pos,
total=total_frames,
mininterval=tqdm_mininterval) as pbar:
for _ in range(min(count_max, total_frames)):
success, frame = cap.read()
mov.append(frame)
pbar.update(1)
cap.release()
image_series_kwargs.update(data=H5DataIO(
DataChunkIterator(
tqdm(iterable=np.array(mov),
desc=
f"Writing movie data for {Path(file).name}",
position=tqdm_pos,
mininterval=tqdm_mininterval),
iter_axis=0, # nwb standard is time as zero axis
maxshape=tuple(maxshape)),
compression="gzip",
chunks=best_gzip_chunk))
if module_name is None:
nwbfile.add_acquisition(ImageSeries(**image_series_kwargs))
else:
get_module(nwbfile=nwbfile,
name=module_name,
description=module_description).add(
ImageSeries(**image_series_kwargs))
def writeRecording(recording, save_path, acquisition_name):
try:
from pynwb import NWBHDF5IO
from pynwb import NWBFile
from pynwb.ecephys import ElectricalSeries
except ModuleNotFoundError:
raise ModuleNotFoundError(
"To use the Nwb extractors, install pynwb: \n\n"
"pip install pynwb\n\n")
M = recording.getNumChannels()
N = recording.getNumFrames()
nwbfile = NWBFile(source='SpikeInterface::NwbRecordingExtractor',
session_description='',
identifier='',
session_start_time=datetime.now(),
experimenter='',
lab='',
institution='',
experiment_description='',
session_id='')
device = nwbfile.create_device(name='device_name',
source="device_source")
eg_name = 'electrode_group_name'
eg_source = "electrode_group_source"
eg_description = "electrode_group_description"
eg_location = "electrode_group_location"
electrode_group = nwbfile.create_electrode_group(
name=eg_name,
source=eg_source,
location=eg_location,
device=device,
description=eg_description)
for m in range(M):
id = m
location = recording.getChannelProperty(m, 'location')
impedence = -1.0
while len(location) < 3:
location = np.append(location, [0])
nwbfile.add_electrode(id,
x=float(location[0]),
y=float(location[1]),
z=float(location[2]),
imp=impedence,
location='electrode_location',
filtering='none',
group=electrode_group,
description='electrode_description')
electrode_table_region = nwbfile.create_electrode_table_region(
list(range(M)), 'electrode_table_region')
rate = recording.getSamplingFrequency() / 1000
ephys_data = recording.getTraces().T
ephys_ts = ElectricalSeries(
name=acquisition_name,
source='acquisition_source',
data=ephys_data,
electrodes=electrode_table_region,
starting_time=recording.frameToTime(0),
rate=rate,
resolution=1e-6,
comments='Generated from SpikeInterface::NwbRecordingExtractor',
description='acquisition_description')
nwbfile.add_acquisition(ephys_ts)
if os.path.exists(save_path):
os.remove(save_path)
with NWBHDF5IO(save_path, 'w') as io:
io.write(nwbfile)
class TestH5DataIO(unittest.TestCase):
"""
Test that H5DataIO functions correctly on round trip with the HDF5IO backend
"""
def setUp(self):
self.nwbfile = NWBFile('a', 'b',
datetime(1970, 1, 1, 12, tzinfo=tzutc()))
self.path = "test_pynwb_io_hdf5_h5dataIO.h5"
def tearDown(self):
if (os.path.exists(self.path)):
os.remove(self.path)
def test_gzip_timestamps(self):
ts = TimeSeries('ts_name', [1, 2, 3],
'A',
timestamps=H5DataIO(np.array([1., 2., 3.]),
compression='gzip'))
self.nwbfile.add_acquisition(ts)
with NWBHDF5IO(self.path, 'w') as io:
io.write(self.nwbfile)
# confirm that the dataset was indeed compressed
infile = File(self.path, 'r')
self.assertEquals(
infile['/acquisition/ts_name/timestamps'].compression, 'gzip')
def test_write_dataset_custom_compress(self):
a = H5DataIO(np.arange(30).reshape(5, 2, 3),
compression='gzip',
compression_opts=5,
shuffle=True,
fletcher32=True)
ts = TimeSeries('ts_name', a, 'A', timestamps=np.arange(5))
self.nwbfile.add_acquisition(ts)
with NWBHDF5IO(self.path, 'w') as io:
io.write(self.nwbfile)
infile = File(self.path, 'r')
dset = infile['/acquisition/ts_name/data']
self.assertTrue(np.all(dset[:] == a.data))
self.assertEqual(dset.compression, 'gzip')
self.assertEqual(dset.compression_opts, 5)
self.assertEqual(dset.shuffle, True)
self.assertEqual(dset.fletcher32, True)
def test_write_dataset_custom_chunks(self):
a = H5DataIO(np.arange(30).reshape(5, 2, 3), chunks=(1, 1, 3))
ts = TimeSeries('ts_name', a, 'A', timestamps=np.arange(5))
self.nwbfile.add_acquisition(ts)
with NWBHDF5IO(self.path, 'w') as io:
io.write(self.nwbfile)
infile = File(self.path, 'r')
dset = infile['/acquisition/ts_name/data']
self.assertTrue(np.all(dset[:] == a.data))
self.assertEqual(dset.chunks, (1, 1, 3))
def test_write_dataset_custom_fillvalue(self):
a = H5DataIO(np.arange(20).reshape(5, 4), fillvalue=-1)
ts = TimeSeries('ts_name', a, 'A', timestamps=np.arange(5))
self.nwbfile.add_acquisition(ts)
with NWBHDF5IO(self.path, 'w') as io:
io.write(self.nwbfile)
infile = File(self.path, 'r')
dset = infile['/acquisition/ts_name/data']
self.assertTrue(np.all(dset[:] == a.data))
self.assertEqual(dset.fillvalue, -1)
def test_write_dataset_datachunkiterator(self):
a = np.arange(30).reshape(5, 2, 3)
aiter = iter(a)
daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2)
ts = TimeSeries('ts_name', daiter, 'A', timestamps=np.arange(5))
self.nwbfile.add_acquisition(ts)
with NWBHDF5IO(self.path, 'w') as io:
io.write(self.nwbfile)
infile = File(self.path, 'r')
dset = infile['/acquisition/ts_name/data']
self.assertListEqual(dset[:].tolist(), a.tolist())
def test_write_dataset_datachunkiterator_with_compression(self):
a = np.arange(30).reshape(5, 2, 3)
aiter = iter(a)
daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2)
wrapped_daiter = H5DataIO(data=daiter,
compression='gzip',
compression_opts=5,
shuffle=True,
fletcher32=True)
ts = TimeSeries('ts_name',
wrapped_daiter,
'A',
timestamps=np.arange(5))
self.nwbfile.add_acquisition(ts)
with NWBHDF5IO(self.path, 'w') as io:
io.write(self.nwbfile)
infile = File(self.path, 'r')
dset = infile['/acquisition/ts_name/data']
self.assertEqual(dset.shape, a.shape)
self.assertListEqual(dset[:].tolist(), a.tolist())
self.assertEqual(dset.compression, 'gzip')
self.assertEqual(dset.compression_opts, 5)
self.assertEqual(dset.shuffle, True)
self.assertEqual(dset.fletcher32, True)
class ABF1Converter:
"""
Converts Neuron2BrainLab's ABF1 files from a single cell (collected without amplifier settings from the
multi-clamp commander) to a collective NeurodataWithoutBorders v2 file.
Modeled after ABFConverter created by the Allen Institute.
Parameters
----------
inputPath: path to ABF file or a folder of ABF files to be converted
outputFilePath: path to the output NWB file
acquisitionChannelName: Allows to output only a specific acquisition channel, defaults to all
stimulusChannelName: Allows to output only a specific stimulus channel,
defaults to all. The name can also be an AD channel name for cases where
the stimulus is recorded as well.
responseGain: user-input float indicating scalar gain for response channel
stimulusGain: user-input float indicating scalar gain for stimulus channel
clampMode: 0 or 1 integer indicating clamp mode (0 is VC, 1 is CC). If not None, overwrites clamp mode provided in ABF file
"""
def __init__(self, inputPath, outputFilePath, acquisitionChannelName=None, stimulusChannelName=None,
responseGain = 1, stimulusGain = 1, responseOffset = 0, clampMode = None):
self.inputPath = inputPath
self.debug=False
if os.path.isfile(self.inputPath):
print(inputPath)
abf = pyabf.ABF(self.inputPath)
if abf.abfVersion["major"] != 1:
raise ValueError(f"The ABF version for the file {abf} is not supported.")
self.fileNames = [os.path.basename(self.inputPath)]
self.abfFiles = [abf]
elif os.path.isdir(self.inputPath):
abfFiles = []
for dirpath, dirnames, filenames in os.walk(self.inputPath):
# Find all .abf files in the directory
if len(dirnames) == 0 and len(glob.glob(dirpath + "/*.abf")) != 0:
abfFiles += glob.glob(dirpath + "/*.abf")
if len(abfFiles) == 0:
raise ValueError(f"{inputPath} contains no ABF Files.")
# Arrange the ABF files in ascending order
abfFiles.sort(key=lambda x: os.path.basename(x))
# Collect file names for description
self.fileNames = []
for file in abfFiles:
self.fileNames += [os.path.basename(file)]
self.abfFiles = []
for abfFile in abfFiles:
# Load each ABF file using pyabf
abf = pyabf.ABF(abfFile)
# Check for ABF version
if abf.abfVersion["major"] != 1:
raise ValueError(f"The ABF version for the file {abf} is not supported.")
self.abfFiles += [abf]
if clampMode:
self.clampMode = clampMode # sometimes the abf-based clamp mode is wrong
else:
self.clampMode = self.abfFiles[0]._headerV1.nExperimentType
self.outputPath = outputFilePath
# Take metadata input, and return hard coded values for None
self.responseGain = responseGain
self.stimulusGain = stimulusGain
self.responseOffset = responseOffset
self.acquisitionChannelName = acquisitionChannelName
self.stimulusChannelName = stimulusChannelName
def _outputMetadata(self):
"""
Create metadata files in HTML format next to the existing ABF files.
"""
for abfFile in self.abfFiles:
root, ext = os.path.splitext(abfFile.abfFilePath)
pyabf.abfHeaderDisplay.abfInfoPage(abfFile).generateHTML(saveAs=root + ".html")
def _getComments(self, abf):
"""
Accesses the tag comments created in Clampfit
"""
return abf.tagComments
def _createNWBFile(self):
"""
Creates the NWB file for the cell, as defined by PyNWB
"""
self.start_time = self.abfFiles[0].abfDateTime
self.inputCellName = os.path.basename(self.inputPath)
creatorInfo = self.abfFiles[0]._headerV1.sCreatorInfo
creatorVersion = self.abfFiles[0]._headerV1.creatorVersionString
experiment_description = (f"{creatorInfo} v{creatorVersion}")
self.NWBFile = NWBFile(
session_description="",
session_start_time=self.start_time,
experiment_description = experiment_description,
identifier=self.inputCellName,
file_create_date= datetime.now(tzlocal()),
experimenter=None,
notes=""
)
return self.NWBFile
def _createDevice(self):
creatorInfo = self.abfFiles[0]._headerV1.sCreatorInfo
creatorVersion = self.abfFiles[0]._headerV1.creatorVersionString
self.device = self.NWBFile.create_device(name=f"{creatorInfo} {creatorVersion}")
def _createElectrode(self):
self.electrode = self.NWBFile.create_ic_electrode(name='elec0', device=self.device, description='PLACEHOLDER')
def _unitConversion(self, unit):
# Returns a 2-list of base unit and conversion factor
if unit == 'V':
return 1.0, 'V'
elif unit == 'mV':
return 1e-3, 'V'
elif unit == 'A':
return 1.0, 'A'
elif unit == 'pA':
return 1e-12, 'A'
elif unit == 'nA':
return 1e-9, 'A'
else:
# raise ValueError(f"{unit} is not a valid unit.")
return 1.0, 'V' # hard coded for units stored as '?'
def _getClampMode(self):
"""
Returns the clamp mode of the experiment.
Voltage Clamp Mode = 0
Current Clamp Mode = 1
"""
return self.clampMode
def _addStimulus(self):
"""
Adds a stimulus class as defined by PyNWB to the NWB File.
Written for experiments conducted from a single channel.
For multiple channels, refer to https://github.com/AllenInstitute/ipfx/blob/master/ipfx/x_to_nwb/ABFConverter.py
"""
for idx, abfFile in enumerate(self.abfFiles):
isStimulus = True
if self.stimulusChannelName is None:
channelList = abfFile.adcNames
channelIndices = range(len(channelList))
else:
if self.stimulusChannelName in abfFile.dacNames:
channelList = abfFile.dacNames
channelIndices = [channelList.index(self.stimulusChannelName)]
elif self.stimulusChannelName in abfFile.adcNames:
isStimulus = False
channelList = abfFile.adcNames
channelIndices = [channelList.index(self.stimulusChannelName)]
else:
raise ValueError(f"Channel {self.stimulusChannelName} could not be found.")
for i in range(abfFile.sweepCount):
for channelIndex in channelIndices:
if self.debug:
print(f"stimulus: abfFile={abfFile.abfFilePath}, sweep={i}, channelIndex={channelIndex}, channelName={channelList[channelIndex]}")
# Collect data from pyABF
abfFile.setSweep(i, channel=channelIndex)
seriesName = f"Index_{idx}_{i}_{channelIndex}"
if isStimulus:
data = abfFile.sweepC
scaledUnit = abfFile.sweepUnitsC
else:
data = abfFile.sweepY
scaledUnit = abfFile.sweepUnitsY
stimulusGain = self.stimulusGain
data = data * stimulusGain
conversion, unit = self._unitConversion(scaledUnit)
electrode = self.electrode
resolution = np.nan
starting_time = 0.0
rate = float(abfFile.dataRate)
# Create a JSON file for the description field
description = json.dumps({"file_name": os.path.basename(self.fileNames[idx]),
"file_version": abfFile.abfVersionString,
"sweep_number": i,
"protocol": abfFile.protocol,
"protocol_path": abfFile.protocolPath,
"comments": self._getComments(abfFile)},
sort_keys=True, indent=4)
# Determine the clamp mode
if self.clampMode == 0:
stimulusClass = VoltageClampStimulusSeries
elif self.clampMode == 1:
stimulusClass = CurrentClampStimulusSeries
else:
raise ValueError(f"Unsupported clamp mode {self.clampMode}")
data = createCompressedDataset(data)
# Create a stimulus class
stimulus = stimulusClass(name=seriesName,
data=data,
sweep_number=i,
electrode=electrode,
gain=stimulusGain,
resolution=resolution,
conversion=conversion,
starting_time=starting_time,
rate=rate,
unit=unit,
description=description
)
self.NWBFile.add_stimulus(stimulus)
def _addAcquisition(self):
"""
Adds an acquisition class as defined by PyNWB to the NWB File.
Written for experiments conducted from a single channel.
For multiple channels, refer to https://github.com/AllenInstitute/ipfx/blob/master/ipfx/x_to_nwb/ABFConverter.py
"""
for idx, abfFile in enumerate(self.abfFiles):
if self.acquisitionChannelName is None:
channelList = abfFile.adcNames
channelIndices = range(len(channelList))
else:
if self.acquisitionChannelName in abfFile.adcNames:
channelList = abfFile.adcNames
channelIndices = [channelList.index(self.acquisitionChannelName)]
else:
raise ValueError(f"Channel {self.acquisitionChannelName} could not be found.")
for i in range(abfFile.sweepCount):
for channelIndex in channelIndices:
if self.debug:
print(f"acquisition: abfFile={abfFile.abfFilePath}, sweep={i}, channelIndex={channelIndex}, channelName={channelList[channelIndex]}")
# Collect data from pyABF
abfFile.setSweep(i, channel=channelIndex)
seriesName = f"Index_{idx}_{i}_{channelIndex}"
responseGain = self.responseGain
responseOffset = self.responseOffset
data = abfFile.sweepY * responseGain + responseOffset
conversion, unit = self._unitConversion(abfFile.sweepUnitsY)
electrode = self.electrode
resolution = np.nan
starting_time = 0.0
rate = float(abfFile.dataRate)
# Create a JSON file for the description field
description = json.dumps({"file_name": os.path.basename(self.fileNames[idx]),
"file_version": abfFile.abfVersionString,
"sweep_number": i,
"protocol": abfFile.protocol,
"protocol_path": abfFile.protocolPath,
"comments": self._getComments(abfFile)},
sort_keys=True, indent=4)
# Create an acquisition class
# Note: voltage input produces current output; current input produces voltage output
data = createCompressedDataset(data)
if self.clampMode == 1:
acquisition = CurrentClampSeries(name=seriesName,
data=data,
sweep_number=i,
electrode=electrode,
gain=responseGain,
resolution=resolution,
conversion=conversion,
starting_time=starting_time,
rate=rate,
unit=unit,
description=description,
bias_current=np.nan,
bridge_balance=np.nan,
capacitance_compensation=np.nan,
)
elif self.clampMode == 0:
acquisition = VoltageClampSeries(name=seriesName,
data=data,
sweep_number=i,
electrode=electrode,
gain=responseGain,
resolution=resolution,
conversion=conversion,
starting_time=starting_time,
rate=rate,
unit=unit,
description=description,
capacitance_fast=np.nan,
capacitance_slow=np.nan,
resistance_comp_bandwidth=np.nan,
resistance_comp_correction=np.nan,
resistance_comp_prediction=np.nan,
whole_cell_capacitance_comp=np.nan,
whole_cell_series_resistance_comp=np.nan
)
else:
raise ValueError(f"Unsupported clamp mode {self.clampMode}")
self.NWBFile.add_acquisition(acquisition)
def convert(self):
"""
Iterates through the functions in the specified order.
:return: True (for success)
"""
self._createNWBFile()
self._createDevice()
self._createElectrode()
self._getClampMode()
self._addStimulus()
self._addAcquisition()
with NWBHDF5IO(self.outputPath, "w") as io:
io.write(self.NWBFile, cache_spec=True)
print(f"Successfully converted to {self.outputPath}.")
filename1 = 'external1_example.nwb'
filename2 = 'external2_example.nwb'
filename3 = 'external_linkcontainer_example.nwb'
filename4 = 'external_linkdataset_example.nwb'
# Create the first file
nwbfile1 = NWBFile(session_description='demonstrate external files',
identifier='NWBE1',
session_start_time=start_time,
file_create_date=create_date)
# Create the second file
test_ts1 = TimeSeries(name='test_timeseries1',
data=data,
unit='SIunit',
timestamps=timestamps)
nwbfile1.add_acquisition(test_ts1)
# Write the first file
io = NWBHDF5IO(filename1, 'w')
io.write(nwbfile1)
io.close()
# Create the second file
nwbfile2 = NWBFile(session_description='demonstrate external files',
identifier='NWBE2',
session_start_time=start_time,
file_create_date=create_date)
# Create the second file
test_ts2 = TimeSeries(name='test_timeseries2',
data=data,
unit='SIunit',
timestamps=timestamps)
def export_to_nwb(session_key,
nwb_output_dir=default_nwb_output_dir,
save=False,
overwrite=True):
this_session = (acquisition.Session & session_key).fetch1()
# =============== General ====================
# -- NWB file - a NWB2.0 file for each session
nwbfile = NWBFile(session_description=this_session['session_note'],
identifier='_'.join([
this_session['subject_id'],
this_session['session_time'].strftime('%Y-%m-%d'),
this_session['session_id']
]),
session_start_time=this_session['session_time'],
file_create_date=datetime.now(tzlocal()),
experimenter='; '.join(
(acquisition.Session.Experimenter
& session_key).fetch('experimenter')),
institution=institution,
experiment_description=experiment_description,
related_publications=related_publications,
keywords=keywords)
# -- subject
subj = (subject.Subject & session_key).fetch1()
nwbfile.subject = pynwb.file.Subject(
subject_id=this_session['subject_id'],
description=subj['subject_description'],
genotype=' x '.join(
(subject.Subject.Allele & session_key).fetch('allele')),
sex=subj['sex'],
species=subj['species'])
# =============== Intracellular ====================
cell = ((intracellular.Cell
& session_key).fetch1() if len(intracellular.Cell
& session_key) == 1 else None)
if cell:
# metadata
whole_cell_device = nwbfile.create_device(name=cell['device_name'])
ic_electrode = nwbfile.create_ic_electrode(
name=cell['cell_id'],
device=whole_cell_device,
description='N/A',
filtering='N/A',
location='; '.join([
f'{k}: {str(v)}'
for k, v in dict((reference.BrainLocation & cell).fetch1(),
depth=cell['recording_depth']).items()
]))
# acquisition - membrane potential
mp, mp_timestamps = (intracellular.MembranePotential & cell).fetch1(
'membrane_potential', 'membrane_potential_timestamps')
nwbfile.add_acquisition(
pynwb.icephys.PatchClampSeries(name='PatchClampSeries',
electrode=ic_electrode,
unit='mV',
conversion=1e-3,
gain=1.0,
data=mp,
timestamps=mp_timestamps))
# acquisition - spike train
spk, spk_timestamps = (intracellular.SpikeTrain & cell).fetch1(
'spike_train', 'spike_timestamps')
nwbfile.add_acquisition(
pynwb.icephys.PatchClampSeries(name='SpikeTrain',
electrode=ic_electrode,
unit='a.u.',
conversion=1e1,
gain=1.0,
data=spk,
timestamps=spk_timestamps))
# =============== Behavior ====================
behavior_data = ((behavior.Behavior & session_key).fetch1()
if len(behavior.Behavior & session_key) == 1 else None)
if behavior_data:
behav_acq = pynwb.behavior.BehavioralTimeSeries(name='behavior')
nwbfile.add_acquisition(behav_acq)
[behavior_data.pop(k) for k in behavior.Behavior.primary_key]
timestamps = behavior_data.pop('behavior_timestamps')
# get behavior data description from the comments of table definition
behavior_descriptions = {
attr:
re.search(f'(?<={attr})(.*)#(.*)',
str(behavior.Behavior.heading)).groups()[-1].strip()
for attr in behavior_data
}
for b_k, b_v in behavior_data.items():
behav_acq.create_timeseries(name=b_k,
description=behavior_descriptions[b_k],
unit='a.u.',
conversion=1.0,
data=b_v,
timestamps=timestamps)
# =============== Photostimulation ====================
photostim = ((stimulation.PhotoStimulation
& session_key).fetch1() if len(stimulation.PhotoStimulation
& session_key) == 1 else None)
if photostim:
photostim_device = (stimulation.PhotoStimDevice & photostim).fetch1()
stim_device = nwbfile.create_device(
name=photostim_device['device_name'])
stim_site = pynwb.ogen.OptogeneticStimulusSite(
name='-'.join([photostim['hemisphere'],
photostim['brain_region']]),
device=stim_device,
excitation_lambda=float(
(stimulation.PhotoStimulationProtocol
& photostim).fetch1('photo_stim_excitation_lambda')),
location='; '.join([
f'{k}: {str(v)}' for k, v in (reference.ActionLocation
& photostim).fetch1().items()
]),
description=(stimulation.PhotoStimulationProtocol
& photostim).fetch1('photo_stim_notes'))
nwbfile.add_ogen_site(stim_site)
if photostim['photostim_timeseries'] is not None:
nwbfile.add_stimulus(
pynwb.ogen.OptogeneticSeries(
name='_'.join([
'photostim_on',
photostim['photostim_datetime'].strftime(
'%Y-%m-%d_%H-%M-%S')
]),
site=stim_site,
resolution=0.0,
conversion=1e-3,
data=photostim['photostim_timeseries'],
starting_time=photostim['photostim_start_time'],
rate=photostim['photostim_sampling_rate']))
# =============== TrialSet ====================
# NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes:
# 'id', 'start_time' and 'stop_time'.
# Other trial-related information needs to be added in to the trial-table as additional columns (with column name
# and column description)
if len((acquisition.TrialSet & session_key).fetch()) == 1:
# Get trial descriptors from TrialSet.Trial and TrialStimInfo - remove '_trial' prefix (if any)
trial_columns = [{
'name':
tag.replace('trial_', ''),
'description':
re.search(
f'(?<={tag})(.*)#(.*)',
str((acquisition.TrialSet.Trial *
stimulation.TrialPhotoStimInfo
).heading)).groups()[-1].strip()
} for tag in acquisition.TrialSet.Trial.heading.names
if tag not in acquisition.TrialSet.Trial.primary_key +
['start_time', 'stop_time']]
# Trial Events - discard 'trial_start' and 'trial_stop' as we already have start_time and stop_time
# also add `_time` suffix to all events
trial_events = set(((acquisition.TrialSet.EventTime & session_key) -
[{
'trial_event': 'trial_start'
}, {
'trial_event': 'trial_stop'
}]).fetch('trial_event'))
event_names = [{
'name': e + '_time',
'description': d
} for e, d in zip(*(reference.ExperimentalEvent & [{
'event': k
} for k in trial_events]).fetch('event', 'description'))]
# Add new table columns to nwb trial-table for trial-label
for c in trial_columns + event_names:
nwbfile.add_trial_column(**c)
# Add entry to the trial-table
for trial in (acquisition.TrialSet.Trial
& session_key).fetch(as_dict=True):
events = dict(
zip(*(acquisition.TrialSet.EventTime & trial
& [{
'trial_event': e
} for e in trial_events]
).fetch('trial_event', 'event_time')))
# shift event times to be relative to session_start (currently relative to trial_start)
events = {k: v + trial['start_time'] for k, v in events.items()}
trial_tag_value = {**trial, **events}
# rename 'trial_id' to 'id'
trial_tag_value['id'] = trial_tag_value['trial_id']
[
trial_tag_value.pop(k)
for k in acquisition.TrialSet.Trial.primary_key
]
# Final tweaks: i) add '_time' suffix and ii) remove 'trial_' prefix
events = {k + '_time': trial_tag_value.pop(k) for k in events}
trial_attrs = {
k.replace('trial_', ''): trial_tag_value.pop(k)
for k in
[n for n in trial_tag_value if n.startswith('trial_')]
}
nwbfile.add_trial(**trial_tag_value, **events, **trial_attrs)
# =============== Write NWB 2.0 file ===============
if save:
save_file_name = ''.join([nwbfile.identifier, '.nwb'])
if not os.path.exists(nwb_output_dir):
os.makedirs(nwb_output_dir)
if not overwrite and os.path.exists(
os.path.join(nwb_output_dir, save_file_name)):
return nwbfile
with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name),
mode='w') as io:
io.write(nwbfile)
print(f'Write NWB 2.0 file: {save_file_name}')
return nwbfile
device = nwbfile.create_device(device_label)
electrode_group = nwbfile.create_electrode_group(name=device_label +
' electrode group',
description=' ',
device=device,
location='unknown')
nwbfile.add_electrode(i,
x,
y,
z,
imp=np.nan,
location='unknown',
filtering='unknown',
description=label,
group=electrode_group)
electrode_table_region = nwbfile.create_electrode_table_region(
list(range(len(electrode_positions))), 'all ECoG electrodes')
nwbfile.add_acquisition(
LFP(electrical_series=ElectricalSeries('lfp',
'lfp signal for all electrodes',
lfp,
electrode_table_region,
starting_time=0.0,
rate=lfp_rate)))
with NWBHDF5IO('resting_state.nwb') as io:
io.write(nwbfile)
ts = TetrodeSeries(
'test_ephys_data',
'an hypothetical source',
data,
electrode_table_region,
timestamps=timestamps,
trode_id=1,
# Alternatively, could specify starting_time and rate as follows
# starting_time=ephys_timestamps[0],
# rate=rate,
resolution=0.001,
comments=
"This data was randomly generated with numpy, using 1234 as the seed",
description="Random numbers generated with numpy.random.rand")
nwbfile.add_acquisition(ts)
####################
# .. note::
#
# For more information on writing :py:class:`~pynwb.ecephys.ElectricalSeries`,
# see :ref:`ecephys_tutorial`.
#
# Now that we have some data, lets write our file while caching our spec:
from pynwb import NWBHDF5IO
io = NWBHDF5IO('cache_spec_example.nwb', mode='w')
io.write(nwbfile, cache_spec=True)
io.close()
def main():
import os.path
# prerequisites: start
import numpy as np
rate = 10.0
np.random.seed(1234)
data_len = 1000
ephys_data = np.random.rand(data_len)
ephys_timestamps = np.arange(data_len) / rate
spatial_timestamps = ephys_timestamps[::10]
spatial_data = np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T
# prerequisites: end
# create-nwbfile: start
from datetime import datetime
from pynwb import NWBFile
f = NWBFile('the PyNWB tutorial', 'my first synthetic recording', 'EXAMPLE_ID', datetime.now(),
experimenter='Dr. Bilbo Baggins',
lab='Bag End Laboratory',
institution='University of Middle Earth at the Shire',
experiment_description='I went on an adventure with thirteen dwarves to reclaim vast treasures.',
session_id='LONELYMTN')
# create-nwbfile: end
# save-nwbfile: start
from pynwb import get_manager
from pynwb.form.backends.hdf5 import HDF5IO
filename = "example.h5"
io = HDF5IO(filename, manager=get_manager(), mode='w')
io.write(f)
io.close()
# save-nwbfile: end
os.remove(filename)
# create-device: start
device = f.create_device(name='trodes_rig123', source="a source")
# create-device: end
# create-electrode-groups: start
electrode_name = 'tetrode1'
source = "an hypothetical source"
description = "an example tetrode"
location = "somewhere in the hippocampus"
electrode_group = f.create_electrode_group(electrode_name,
source=source,
description=description,
location=location,
device=device)
# create-electrode-groups: end
# create-electrode-table-region: start
for idx in [1, 2, 3, 4]:
f.add_electrode(idx,
x=1.0, y=2.0, z=3.0,
imp=float(-idx),
location='CA1', filtering='none',
description='channel %s' % idx, group=electrode_group)
electrode_table_region = f.create_electrode_table_region([0, 2], 'the first and third electrodes')
# create-electrode-table-region: end
# create-timeseries: start
from pynwb.ecephys import ElectricalSeries
from pynwb.behavior import SpatialSeries
ephys_ts = ElectricalSeries('test_ephys_data',
'an hypothetical source',
ephys_data,
electrode_table_region,
timestamps=ephys_timestamps,
# Alternatively, could specify starting_time and rate as follows
# starting_time=ephys_timestamps[0],
# rate=rate,
resolution=0.001,
comments="This data was randomly generated with numpy, using 1234 as the seed",
description="Random numbers generated with numpy.random.rand")
f.add_acquisition(ephys_ts)
spatial_ts = SpatialSeries('test_spatial_timeseries',
'a stumbling rat',
spatial_data,
'origin on x,y-plane',
timestamps=spatial_timestamps,
resolution=0.1,
comments="This data was generated with numpy, using 1234 as the seed",
description="This 2D Brownian process generated with "
"np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T")
f.add_acquisition(spatial_ts)
# create-timeseries: end
# create-data-interface: start
from pynwb.ecephys import LFP
from pynwb.behavior import Position
lfp = f.add_acquisition(LFP('a hypothetical source'))
ephys_ts = lfp.create_electrical_series('test_ephys_data',
'an hypothetical source',
ephys_data,
electrode_table_region,
timestamps=ephys_timestamps,
resolution=0.001,
comments="This data was randomly generated with numpy, using 1234 as the seed", # noqa: E501
description="Random numbers generated with numpy.random.rand")
pos = f.add_acquisition(Position('a hypothetical source'))
spatial_ts = pos.create_spatial_series('test_spatial_timeseries',
'a stumbling rat',
spatial_data,
'origin on x,y-plane',
timestamps=spatial_timestamps,
resolution=0.1,
comments="This data was generated with numpy, using 1234 as the seed",
description="This 2D Brownian process generated with "
"np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T") # noqa: E501
# create-data-interface: end
# create-epochs: start
epoch_tags = ('example_epoch',)
f.create_epoch(name='epoch1', start_time=0.0, stop_time=1.0, tags=epoch_tags,
description="the first test epoch", timeseries=[ephys_ts, spatial_ts])
f.create_epoch(name='epoch2', start_time=0.0, stop_time=1.0, tags=epoch_tags,
description="the second test epoch", timeseries=[ephys_ts, spatial_ts])
# create-epochs: end
# create-compressed-timeseries: start
from pynwb.ecephys import ElectricalSeries
from pynwb.behavior import SpatialSeries
from pynwb.form.backends.hdf5 import H5DataIO
ephys_ts = ElectricalSeries('test_compressed_ephys_data',
'an hypothetical source',
H5DataIO(ephys_data, compress=True),
electrode_table_region,
timestamps=H5DataIO(ephys_timestamps, compress=True),
resolution=0.001,
comments="This data was randomly generated with numpy, using 1234 as the seed",
description="Random numbers generated with numpy.random.rand")
f.add_acquisition(ephys_ts)
spatial_ts = SpatialSeries('test_compressed_spatial_timeseries',
'a stumbling rat',
H5DataIO(spatial_data, compress=True),
'origin on x,y-plane',
timestamps=H5DataIO(spatial_timestamps, compress=True),
resolution=0.1,
comments="This data was generated with numpy, using 1234 as the seed",
description="This 2D Brownian process generated with "
"np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T")
f.add_acquisition(spatial_ts)
nwbfile = NWBFile(session_start_time=session_start_time, identifier=this_dir,
session_description='unknown')
device = nwbfile.create_device(name='Neuronexus Probe Buzsaki32/H32Package')
group = nwbfile.create_electrode_group(name='all_channels_group',
description='all channels',
device=device,
location='unknown')
for i in range(nchannels):
nwbfile.add_electrode(np.nan, np.nan, np.nan, # position
imp=np.nan,
location='unknown',
filtering='unknown',
group=group)
electrode_table_region = nwbfile.create_electrode_table_region(
list(range(nchannels)), 'all electrodes')
electrical_series = ElectricalSeries(data=amp_data,
rate=amp_fs,
electrodes=electrode_table_region,
name='amp_data')
nwbfile.add_acquisition(LFP(name='amp_data', electrical_series=electrical_series))
nwbfile.add_acquisition(TimeSeries('auxiliary', data=aux_data, rate=amp_fs, unit='na'))
nwbfile.add_acquisition(TimeSeries('supply', data=supply_data, rate=amp_fs, unit='na'))
out_fname = this_dir + '.nwb'
with NWBHDF5IO(out_fname, 'w') as io:
io.write(nwbfile)
def yuta2nwb(session_path='/Users/bendichter/Desktop/Buzsaki/SenzaiBuzsaki2017/YutaMouse41/YutaMouse41-150903',
subject_xls=None, include_spike_waveforms=True, stub=True, cache_spec=True):
subject_path, session_id = os.path.split(session_path)
fpath_base = os.path.split(subject_path)[0]
identifier = session_id
mouse_number = session_id[9:11]
if '-' in session_id:
subject_id, date_text = session_id.split('-')
b = False
else:
subject_id, date_text = session_id.split('b')
b = True
if subject_xls is None:
subject_xls = os.path.join(subject_path, 'YM' + mouse_number + ' exp_sheet.xlsx')
else:
if not subject_xls[-4:] == 'xlsx':
subject_xls = os.path.join(subject_xls, 'YM' + mouse_number + ' exp_sheet.xlsx')
session_start_time = dateparse(date_text, yearfirst=True)
df = pd.read_excel(subject_xls)
subject_data = {}
for key in ['genotype', 'DOB', 'implantation', 'Probe', 'Surgery', 'virus injection', 'mouseID']:
names = df.iloc[:, 0]
if key in names.values:
subject_data[key] = df.iloc[np.argmax(names == key), 1]
if isinstance(subject_data['DOB'], datetime):
age = session_start_time - subject_data['DOB']
else:
age = None
subject = Subject(subject_id=subject_id, age=str(age),
genotype=subject_data['genotype'],
species='mouse')
nwbfile = NWBFile(session_description='mouse in open exploration and theta maze',
identifier=identifier,
session_start_time=session_start_time.astimezone(),
file_create_date=datetime.now().astimezone(),
experimenter='Yuta Senzai',
session_id=session_id,
institution='NYU',
lab='Buzsaki',
subject=subject,
related_publications='DOI:10.1016/j.neuron.2016.12.011')
print('reading and writing raw position data...', end='', flush=True)
ns.add_position_data(nwbfile, session_path)
shank_channels = ns.get_shank_channels(session_path)[:8]
all_shank_channels = np.concatenate(shank_channels)
print('setting up electrodes...', end='', flush=True)
hilus_csv_path = os.path.join(fpath_base, 'early_session_hilus_chans.csv')
lfp_channel = get_reference_elec(subject_xls, hilus_csv_path, session_start_time, session_id, b=b)
print(lfp_channel)
custom_column = [{'name': 'theta_reference',
'description': 'this electrode was used to calculate LFP canonical bands',
'data': all_shank_channels == lfp_channel}]
ns.write_electrode_table(nwbfile, session_path, custom_columns=custom_column, max_shanks=max_shanks)
print('reading LFPs...', end='', flush=True)
lfp_fs, all_channels_data = ns.read_lfp(session_path, stub=stub)
lfp_data = all_channels_data[:, all_shank_channels]
print('writing LFPs...', flush=True)
# lfp_data[:int(len(lfp_data)/4)]
lfp_ts = ns.write_lfp(nwbfile, lfp_data, lfp_fs, name='lfp',
description='lfp signal for all shank electrodes')
for name, channel in special_electrode_dict.items():
ts = TimeSeries(name=name, description='environmental electrode recorded inline with neural data',
data=all_channels_data[:, channel], rate=lfp_fs, unit='V', conversion=np.nan, resolution=np.nan)
nwbfile.add_acquisition(ts)
# compute filtered LFP
print('filtering LFP...', end='', flush=True)
all_lfp_phases = []
for passband in ('theta', 'gamma'):
lfp_fft = filter_lfp(lfp_data[:, all_shank_channels == lfp_channel].ravel(), lfp_fs, passband=passband)
lfp_phase, _ = hilbert_lfp(lfp_fft)
all_lfp_phases.append(lfp_phase[:, np.newaxis])
data = np.dstack(all_lfp_phases)
print('done.', flush=True)
if include_spike_waveforms:
print('writing waveforms...', end='', flush=True)
nshanks = min((max_shanks, len(ns.get_shank_channels(session_path))))
for shankn in np.arange(nshanks, dtype=int) + 1:
ns.write_spike_waveforms(nwbfile, session_path, shankn, stub=stub)
print('done.', flush=True)
decomp_series = DecompositionSeries(name='LFPDecompositionSeries',
description='Theta and Gamma phase for reference LFP',
data=data, rate=lfp_fs,
source_timeseries=lfp_ts,
metric='phase', unit='radians')
decomp_series.add_band(band_name='theta', band_limits=(4, 10))
decomp_series.add_band(band_name='gamma', band_limits=(30, 80))
check_module(nwbfile, 'ecephys', 'contains processed extracellular electrophysiology data').add_data_interface(decomp_series)
[nwbfile.add_stimulus(x) for x in ns.get_events(session_path)]
# create epochs corresponding to experiments/environments for the mouse
sleep_state_fpath = os.path.join(session_path, '{}--StatePeriod.mat'.format(session_id))
exist_pos_data = any(os.path.isfile(os.path.join(session_path, '{}__{}.mat'.format(session_id, task_type['name'])))
for task_type in task_types)
if exist_pos_data:
nwbfile.add_epoch_column('label', 'name of epoch')
for task_type in task_types:
label = task_type['name']
file = os.path.join(session_path, session_id + '__' + label + '.mat')
if os.path.isfile(file):
print('loading position for ' + label + '...', end='', flush=True)
pos_obj = Position(name=label + '_position')
matin = loadmat(file)
tt = matin['twhl_norm'][:, 0]
exp_times = find_discontinuities(tt)
if 'conversion' in task_type:
conversion = task_type['conversion']
else:
conversion = np.nan
for pos_type in ('twhl_norm', 'twhl_linearized'):
if pos_type in matin:
pos_data_norm = matin[pos_type][:, 1:]
spatial_series_object = SpatialSeries(
name=label + '_{}_spatial_series'.format(pos_type),
data=H5DataIO(pos_data_norm, compression='gzip'),
reference_frame='unknown', conversion=conversion,
resolution=np.nan,
timestamps=H5DataIO(tt, compression='gzip'))
pos_obj.add_spatial_series(spatial_series_object)
check_module(nwbfile, 'behavior', 'contains processed behavioral data').add_data_interface(pos_obj)
for i, window in enumerate(exp_times):
nwbfile.add_epoch(start_time=window[0], stop_time=window[1],
label=label + '_' + str(i))
print('done.')
# there are occasional mismatches between the matlab struct and the neuroscope files
# regions: 3: 'CA3', 4: 'DG'
df_unit_features = get_UnitFeatureCell_features(fpath_base, session_id, session_path)
celltype_names = []
for celltype_id, region_id in zip(df_unit_features['fineCellType'].values,
df_unit_features['region'].values):
if celltype_id == 1:
if region_id == 3:
celltype_names.append('pyramidal cell')
elif region_id == 4:
celltype_names.append('granule cell')
else:
raise Exception('unknown type')
elif not np.isfinite(celltype_id):
celltype_names.append('missing')
else:
celltype_names.append(celltype_dict[celltype_id])
custom_unit_columns = [
{
'name': 'cell_type',
'description': 'name of cell type',
'data': celltype_names},
{
'name': 'global_id',
'description': 'global id for cell for entire experiment',
'data': df_unit_features['unitID'].values},
{
'name': 'max_electrode',
'description': 'electrode that has the maximum amplitude of the waveform',
'data': get_max_electrodes(nwbfile, session_path),
'table': nwbfile.electrodes
}]
ns.add_units(nwbfile, session_path, custom_unit_columns, max_shanks=max_shanks)
trialdata_path = os.path.join(session_path, session_id + '__EightMazeRun.mat')
if os.path.isfile(trialdata_path):
trials_data = loadmat(trialdata_path)['EightMazeRun']
trialdatainfo_path = os.path.join(fpath_base, 'EightMazeRunInfo.mat')
trialdatainfo = [x[0] for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0]]
features = trialdatainfo[:7]
features[:2] = 'start_time', 'stop_time',
[nwbfile.add_trial_column(x, 'description') for x in features[4:] + ['condition']]
for trial_data in trials_data:
if trial_data[3]:
cond = 'run_left'
else:
cond = 'run_right'
nwbfile.add_trial(start_time=trial_data[0], stop_time=trial_data[1], condition=cond,
error_run=trial_data[4], stim_run=trial_data[5], both_visit=trial_data[6])
"""
mono_syn_fpath = os.path.join(session_path, session_id+'-MonoSynConvClick.mat')
matin = loadmat(mono_syn_fpath)
exc = matin['FinalExcMonoSynID']
inh = matin['FinalInhMonoSynID']
#exc_obj = CatCellInfo(name='excitatory_connections',
# indices_values=[], cell_index=exc[:, 0] - 1, indices=exc[:, 1] - 1)
#module_cellular.add_container(exc_obj)
#inh_obj = CatCellInfo(name='inhibitory_connections',
# indices_values=[], cell_index=inh[:, 0] - 1, indices=inh[:, 1] - 1)
#module_cellular.add_container(inh_obj)
"""
if os.path.isfile(sleep_state_fpath):
matin = loadmat(sleep_state_fpath)['StatePeriod']
table = TimeIntervals(name='states', description='sleep states of animal')
table.add_column(name='label', description='sleep state')
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append({'start_time': row[0], 'stop_time': row[1], 'label': name})
[table.add_row(**row) for row in sorted(data, key=lambda x: x['start_time'])]
check_module(nwbfile, 'behavior', 'contains behavioral data').add_data_interface(table)
if stub:
out_fname = session_path + '_stub.nwb'
else:
out_fname = session_path + '.nwb'
print('writing NWB file...', end='', flush=True)
with NWBHDF5IO(out_fname, mode='w') as io:
io.write(nwbfile, cache_spec=cache_spec)
print('done.')
print('testing read...', end='', flush=True)
# test read
with NWBHDF5IO(out_fname, mode='r') as io:
io.read()
print('done.')
class TestH5DataIO(TestCase):
"""
Test that H5DataIO functions correctly on round trip with the HDF5IO backend
"""
def setUp(self):
self.nwbfile = NWBFile(session_description='a',
identifier='b',
session_start_time=datetime(1970, 1, 1, 12, tzinfo=tzutc()))
self.path = "test_pynwb_io_hdf5_h5dataIO.h5"
def tearDown(self):
remove_test_file(self.path)
def test_gzip_timestamps(self):
ts = TimeSeries(name='ts_name',
data=[1, 2, 3],
unit='A',
timestamps=H5DataIO(np.array([1., 2., 3.]), compression='gzip'))
self.nwbfile.add_acquisition(ts)
with NWBHDF5IO(self.path, 'w') as io:
io.write(self.nwbfile, cache_spec=False)
# confirm that the dataset was indeed compressed
with File(self.path, 'r') as f:
self.assertEqual(f['/acquisition/ts_name/timestamps'].compression, 'gzip')
def test_write_dataset_custom_compress(self):
a = H5DataIO(np.arange(30).reshape(5, 2, 3),
compression='gzip',
compression_opts=5,
shuffle=True,
fletcher32=True)
ts = TimeSeries(name='ts_name', data=a, unit='A', timestamps=np.arange(5.))
self.nwbfile.add_acquisition(ts)
with NWBHDF5IO(self.path, 'w') as io:
io.write(self.nwbfile, cache_spec=False)
with File(self.path, 'r') as f:
dset = f['/acquisition/ts_name/data']
self.assertTrue(np.all(dset[:] == a.data))
self.assertEqual(dset.compression, 'gzip')
self.assertEqual(dset.compression_opts, 5)
self.assertEqual(dset.shuffle, True)
self.assertEqual(dset.fletcher32, True)
def test_write_dataset_custom_chunks(self):
a = H5DataIO(np.arange(30).reshape(5, 2, 3),
chunks=(1, 1, 3))
ts = TimeSeries(name='ts_name', data=a, unit='A', timestamps=np.arange(5.))
self.nwbfile.add_acquisition(ts)
with NWBHDF5IO(self.path, 'w') as io:
io.write(self.nwbfile, cache_spec=False)
with File(self.path, 'r') as f:
dset = f['/acquisition/ts_name/data']
self.assertTrue(np.all(dset[:] == a.data))
self.assertEqual(dset.chunks, (1, 1, 3))
def test_write_dataset_custom_fillvalue(self):
a = H5DataIO(np.arange(20).reshape(5, 4), fillvalue=-1)
ts = TimeSeries(name='ts_name', data=a, unit='A', timestamps=np.arange(5.))
self.nwbfile.add_acquisition(ts)
with NWBHDF5IO(self.path, 'w') as io:
io.write(self.nwbfile, cache_spec=False)
with File(self.path, 'r') as f:
dset = f['/acquisition/ts_name/data']
self.assertTrue(np.all(dset[:] == a.data))
self.assertEqual(dset.fillvalue, -1)
def test_write_dataset_datachunkiterator_data_and_time(self):
a = np.arange(30).reshape(5, 2, 3)
aiter = iter(a)
daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2)
tstamps = np.arange(5.)
tsiter = DataChunkIterator.from_iterable(tstamps)
ts = TimeSeries(name='ts_name', data=daiter, unit='A', timestamps=tsiter)
self.nwbfile.add_acquisition(ts)
with NWBHDF5IO(self.path, 'w') as io:
io.write(self.nwbfile, cache_spec=False)
with File(self.path, 'r') as f:
dset = f['/acquisition/ts_name/data']
self.assertListEqual(dset[:].tolist(), a.tolist())
def test_write_dataset_datachunkiterator_data_only(self):
a = np.arange(30).reshape(5, 2, 3)
aiter = iter(a)
daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2)
tstamps = np.arange(5.)
ts = TimeSeries(name='ts_name', data=daiter, unit='A', timestamps=tstamps)
self.nwbfile.add_acquisition(ts)
with NWBHDF5IO(self.path, 'w') as io:
io.write(self.nwbfile, cache_spec=False)
with File(self.path, 'r') as f:
dset = f['/acquisition/ts_name/data']
self.assertListEqual(dset[:].tolist(), a.tolist())
def test_write_dataset_datachunkiterator_with_compression(self):
a = np.arange(30).reshape(5, 2, 3)
aiter = iter(a)
daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2)
wrapped_daiter = H5DataIO(data=daiter,
compression='gzip',
compression_opts=5,
shuffle=True,
fletcher32=True)
ts = TimeSeries(name='ts_name', data=wrapped_daiter, unit='A', timestamps=np.arange(5.))
self.nwbfile.add_acquisition(ts)
with NWBHDF5IO(self.path, 'w') as io:
io.write(self.nwbfile, cache_spec=False)
with File(self.path, 'r') as f:
dset = f['/acquisition/ts_name/data']
self.assertEqual(dset.shape, a.shape)
self.assertListEqual(dset[:].tolist(), a.tolist())
self.assertEqual(dset.compression, 'gzip')
self.assertEqual(dset.compression_opts, 5)
self.assertEqual(dset.shuffle, True)
self.assertEqual(dset.fletcher32, True)
class TestHDF5Writer(TestCase):
_required_tests = ('test_nwbio', 'test_write_clobber', 'test_write_cache_spec', 'test_write_no_cache_spec')
@property
def required_tests(self):
return self._required_tests
def setUp(self):
self.manager = get_manager()
self.path = "test_pynwb_io_hdf5.nwb"
self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc())
self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal())
self.container = NWBFile(session_description='a test NWB File', identifier='TEST123',
session_start_time=self.start_time, file_create_date=self.create_date)
ts = TimeSeries(name='test_timeseries', data=list(range(100, 200, 10)), unit='SIunit',
timestamps=np.arange(10.), resolution=0.1)
self.container.add_acquisition(ts)
ts_builder = GroupBuilder('test_timeseries',
attributes={'neurodata_type': 'TimeSeries'},
datasets={'data': DatasetBuilder('data', list(range(100, 200, 10)),
attributes={'unit': 'SIunit',
'conversion': 1.0,
'resolution': 0.1}),
'timestamps': DatasetBuilder('timestamps', np.arange(10.),
attributes={'unit': 'seconds',
'interval': 1})})
self.builder = GroupBuilder(
'root', groups={'acquisition': GroupBuilder('acquisition', groups={'test_timeseries': ts_builder}),
'analysis': GroupBuilder('analysis'),
'general': GroupBuilder('general'),
'processing': GroupBuilder('processing'),
'stimulus': GroupBuilder(
'stimulus',
groups={'presentation': GroupBuilder('presentation'),
'templates': GroupBuilder('templates')})},
datasets={'file_create_date': DatasetBuilder('file_create_date', [self.create_date.isoformat()]),
'identifier': DatasetBuilder('identifier', 'TEST123'),
'session_description': DatasetBuilder('session_description', 'a test NWB File'),
'nwb_version': DatasetBuilder('nwb_version', '1.0.6'),
'session_start_time': DatasetBuilder('session_start_time', self.start_time.isoformat())},
attributes={'neurodata_type': 'NWBFile'})
def tearDown(self):
remove_test_file(self.path)
def test_nwbio(self):
with HDF5IO(self.path, manager=self.manager, mode='a') as io:
io.write(self.container)
with File(self.path, 'r') as f:
self.assertIn('acquisition', f)
self.assertIn('analysis', f)
self.assertIn('general', f)
self.assertIn('processing', f)
self.assertIn('file_create_date', f)
self.assertIn('identifier', f)
self.assertIn('session_description', f)
self.assertIn('session_start_time', f)
acq = f.get('acquisition')
self.assertIn('test_timeseries', acq)
def test_write_clobber(self):
with HDF5IO(self.path, manager=self.manager, mode='a') as io:
io.write(self.container)
with self.assertRaisesWith(UnsupportedOperation,
"Unable to open file %s in 'w-' mode. File already exists." % self.path):
with HDF5IO(self.path, manager=self.manager, mode='w-') as io:
pass
def test_write_cache_spec(self):
'''
Round-trip test for writing spec and reading it back in
'''
with HDF5IO(self.path, manager=self.manager, mode="a") as io:
io.write(self.container)
with File(self.path, 'r') as f:
self.assertIn('specifications', f)
ns_catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace)
HDF5IO.load_namespaces(ns_catalog, self.path)
original_ns = self.manager.namespace_catalog.get_namespace('core')
cached_ns = ns_catalog.get_namespace('core')
self.maxDiff = None
for key in ('author', 'contact', 'doc', 'full_name', 'name'):
with self.subTest(namespace_field=key):
self.assertEqual(original_ns[key], cached_ns[key])
for dt in original_ns.get_registered_types():
with self.subTest(neurodata_type=dt):
original_spec = original_ns.get_spec(dt)
cached_spec = cached_ns.get_spec(dt)
with self.subTest(test='data_type spec read back in'):
self.assertIsNotNone(cached_spec)
with self.subTest(test='cached spec preserved original spec'):
self.assertDictEqual(original_spec, cached_spec)
def test_write_no_cache_spec(self):
'''
Round-trip test for not writing spec
'''
with HDF5IO(self.path, manager=self.manager, mode="a") as io:
io.write(self.container, cache_spec=False)
with File(self.path, 'r') as f:
self.assertNotIn('specifications', f)
class NWBFileTest(TestCase):
def setUp(self):
self.start = datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal())
self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc())
self.create = [
datetime(2017, 5, 1, 12, tzinfo=tzlocal()),
datetime(2017, 5, 2, 13, 0, 0, 1, tzinfo=tzutc()),
datetime(2017, 5, 2, 14, tzinfo=tzutc())
]
self.path = 'nwbfile_test.h5'
self.nwbfile = NWBFile(
'a test session description for a test NWBFile',
'FILE123',
self.start,
file_create_date=self.create,
timestamps_reference_time=self.ref_time,
experimenter='A test experimenter',
lab='a test lab',
institution='a test institution',
experiment_description='a test experiment description',
session_id='test1',
notes='my notes',
pharmacology='drugs',
protocol='protocol',
related_publications='my pubs',
slices='my slices',
surgery='surgery',
virus='a virus',
source_script='noscript',
source_script_file_name='nofilename',
stimulus_notes='test stimulus notes',
data_collection='test data collection notes',
keywords=('these', 'are', 'keywords'))
def test_constructor(self):
self.assertEqual(self.nwbfile.session_description,
'a test session description for a test NWBFile')
self.assertEqual(self.nwbfile.identifier, 'FILE123')
self.assertEqual(self.nwbfile.session_start_time, self.start)
self.assertEqual(self.nwbfile.file_create_date, self.create)
self.assertEqual(self.nwbfile.lab, 'a test lab')
self.assertEqual(self.nwbfile.experimenter, ('A test experimenter', ))
self.assertEqual(self.nwbfile.institution, 'a test institution')
self.assertEqual(self.nwbfile.experiment_description,
'a test experiment description')
self.assertEqual(self.nwbfile.session_id, 'test1')
self.assertEqual(self.nwbfile.stimulus_notes, 'test stimulus notes')
self.assertEqual(self.nwbfile.data_collection,
'test data collection notes')
self.assertEqual(self.nwbfile.related_publications, ('my pubs', ))
self.assertEqual(self.nwbfile.source_script, 'noscript')
self.assertEqual(self.nwbfile.source_script_file_name, 'nofilename')
self.assertEqual(self.nwbfile.keywords, ('these', 'are', 'keywords'))
self.assertEqual(self.nwbfile.timestamps_reference_time, self.ref_time)
def test_create_electrode_group(self):
name = 'example_electrode_group'
desc = 'An example electrode'
loc = 'an example location'
d = self.nwbfile.create_device('a fake device')
elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, d)
self.assertEqual(elecgrp.description, desc)
self.assertEqual(elecgrp.location, loc)
self.assertIs(elecgrp.device, d)
def test_create_custom_intervals(self):
df_words = pd.DataFrame({
'start_time': [.1, 2.],
'stop_time': [.8, 2.3],
'label': ['hello', 'there']
})
words = TimeIntervals.from_dataframe(df_words, name='words')
self.nwbfile.add_time_intervals(words)
self.assertEqual(self.nwbfile.intervals['words'], words)
def test_create_electrode_group_invalid_index(self):
"""
Test the case where the user creates an electrode table region with
indexes that are out of range of the amount of electrodes added.
"""
nwbfile = NWBFile('a', 'b', datetime.now(tzlocal()))
device = nwbfile.create_device('a')
elecgrp = nwbfile.create_electrode_group('a',
'b',
device=device,
location='a')
for i in range(4):
nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=i)
with self.assertRaises(IndexError):
nwbfile.create_electrode_table_region(list(range(6)), 'test')
def test_access_group_after_io(self):
"""
Motivated by #739
"""
nwbfile = NWBFile('a', 'b', datetime.now(tzlocal()))
device = nwbfile.create_device('a')
elecgrp = nwbfile.create_electrode_group('a',
'b',
device=device,
location='a')
nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=0)
with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io:
io.write(nwbfile)
with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io:
nwbfile_i = io.read()
for aa, bb in zip(nwbfile_i.electrodes['group'][:],
nwbfile.electrodes['group'][:]):
self.assertEqual(aa.name, bb.name)
for i in range(4):
nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=i + 1)
with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io:
io.write(nwbfile)
with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io:
nwbfile_i = io.read()
for aa, bb in zip(nwbfile_i.electrodes['group'][:],
nwbfile.electrodes['group'][:]):
self.assertEqual(aa.name, bb.name)
remove_test_file("electrodes_mwe.nwb")
def test_access_processing(self):
self.nwbfile.create_processing_module('test_mod', 'test_description')
# test deprecate .modules
with self.assertWarnsWith(
DeprecationWarning,
'NWBFile.modules has been replaced by NWBFile.processing.'):
modules = self.nwbfile.modules['test_mod']
self.assertIs(self.nwbfile.processing['test_mod'], modules)
def test_epoch_tags(self):
tags1 = ['t1', 't2']
tags2 = ['t3', 't4']
tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float64)
ts = TimeSeries("test_ts",
list(range(len(tstamps))),
'unit',
timestamps=tstamps)
expected_tags = tags1 + tags2
self.nwbfile.add_epoch(0.0, 1.0, tags1, ts)
self.nwbfile.add_epoch(0.0, 1.0, tags2, ts)
tags = self.nwbfile.epoch_tags
self.assertEqual(set(expected_tags), set(tags))
def test_add_acquisition(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.acquisition), 1)
def test_add_stimulus(self):
self.nwbfile.add_stimulus(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.stimulus), 1)
def test_add_stimulus_template(self):
self.nwbfile.add_stimulus_template(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.stimulus_template), 1)
def test_add_analysis(self):
self.nwbfile.add_analysis(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.analysis), 1)
def test_add_acquisition_check_dups(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
with self.assertRaises(ValueError):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
def test_get_acquisition_empty(self):
with self.assertRaisesWith(ValueError,
"acquisition of NWBFile 'root' is empty."):
self.nwbfile.get_acquisition()
def test_get_acquisition_multiple_elements(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.nwbfile.add_acquisition(
TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
msg = "More than one element in acquisition of NWBFile 'root' -- must specify a name."
with self.assertRaisesWith(ValueError, msg):
self.nwbfile.get_acquisition()
def test_add_acquisition_invalid_name(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
msg = "\"'TEST_TS' not found in acquisition of NWBFile 'root'.\""
with self.assertRaisesWith(KeyError, msg):
self.nwbfile.get_acquisition("TEST_TS")
def test_set_electrode_table(self):
table = ElectrodeTable()
dev1 = self.nwbfile.create_device('dev1')
group = self.nwbfile.create_electrode_group('tetrode1',
'tetrode description',
'tetrode location', dev1)
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-1.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-2.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-3.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-4.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
self.nwbfile.set_electrode_table(table)
self.assertIs(self.nwbfile.electrodes, table)
self.assertIs(table.parent, self.nwbfile)
def test_add_unit_column(self):
self.nwbfile.add_unit_column('unit_type', 'the type of unit')
self.assertEqual(self.nwbfile.units.colnames, ('unit_type', ))
def test_add_unit(self):
self.nwbfile.add_unit(id=1)
self.assertEqual(len(self.nwbfile.units), 1)
self.nwbfile.add_unit(id=2)
self.nwbfile.add_unit(id=3)
self.assertEqual(len(self.nwbfile.units), 3)
def test_add_trial_column(self):
self.nwbfile.add_trial_column('trial_type', 'the type of trial')
self.assertEqual(self.nwbfile.trials.colnames,
('start_time', 'stop_time', 'trial_type'))
def test_add_trial(self):
self.nwbfile.add_trial(start_time=10.0, stop_time=20.0)
self.assertEqual(len(self.nwbfile.trials), 1)
self.nwbfile.add_trial(start_time=30.0, stop_time=40.0)
self.nwbfile.add_trial(start_time=50.0, stop_time=70.0)
self.assertEqual(len(self.nwbfile.trials), 3)
def test_add_invalid_times_column(self):
self.nwbfile.add_invalid_times_column(
'comments', 'description of reason for omitting time')
self.assertEqual(self.nwbfile.invalid_times.colnames,
('start_time', 'stop_time', 'comments'))
def test_add_invalid_time_interval(self):
self.nwbfile.add_invalid_time_interval(start_time=0.0, stop_time=12.0)
self.assertEqual(len(self.nwbfile.invalid_times), 1)
self.nwbfile.add_invalid_time_interval(start_time=15.0, stop_time=16.0)
self.nwbfile.add_invalid_time_interval(start_time=17.0, stop_time=20.5)
self.assertEqual(len(self.nwbfile.invalid_times), 3)
def test_add_invalid_time_w_ts(self):
ts = TimeSeries(name='name', data=[1.2], rate=1.0, unit='na')
self.nwbfile.add_invalid_time_interval(start_time=18.0,
stop_time=20.6,
timeseries=ts,
tags=('hi', 'there'))
def test_add_electrode(self):
dev1 = self.nwbfile.create_device(name='dev1')
group = self.nwbfile.create_electrode_group(
name='tetrode1',
description='tetrode description',
location='tetrode location',
device=dev1)
self.nwbfile.add_electrode(x=1.0,
y=2.0,
z=3.0,
imp=-1.0,
location='CA1',
filtering='none',
group=group,
id=1)
elec = self.nwbfile.electrodes[0]
self.assertEqual(elec.index[0], 1)
self.assertEqual(elec.iloc[0]['x'], 1.0)
self.assertEqual(elec.iloc[0]['y'], 2.0)
self.assertEqual(elec.iloc[0]['z'], 3.0)
self.assertEqual(elec.iloc[0]['location'], 'CA1')
self.assertEqual(elec.iloc[0]['filtering'], 'none')
self.assertEqual(elec.iloc[0]['group'], group)
def test_add_electrode_some_opt(self):
dev1 = self.nwbfile.create_device(name='dev1')
group = self.nwbfile.create_electrode_group(
name='tetrode1',
description='tetrode description',
location='tetrode location',
device=dev1)
self.nwbfile.add_electrode(x=1.0,
y=2.0,
z=3.0,
imp=-1.0,
location='CA1',
filtering='none',
group=group,
id=1,
rel_x=4.0,
rel_y=5.0,
rel_z=6.0,
reference='ref1')
self.nwbfile.add_electrode(x=1.0,
y=2.0,
z=3.0,
imp=-1.0,
location='CA1',
filtering='none',
group=group,
id=2,
rel_x=7.0,
rel_y=8.0,
rel_z=9.0,
reference='ref2')
elec = self.nwbfile.electrodes[0]
self.assertEqual(elec.iloc[0]['rel_x'], 4.0)
self.assertEqual(elec.iloc[0]['rel_y'], 5.0)
self.assertEqual(elec.iloc[0]['rel_z'], 6.0)
self.assertEqual(elec.iloc[0]['reference'], 'ref1')
elec = self.nwbfile.electrodes[1]
self.assertEqual(elec.iloc[0]['rel_x'], 7.0)
self.assertEqual(elec.iloc[0]['rel_y'], 8.0)
self.assertEqual(elec.iloc[0]['rel_z'], 9.0)
self.assertEqual(elec.iloc[0]['reference'], 'ref2')
def test_all_children(self):
ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
self.nwbfile.add_acquisition(ts1)
self.nwbfile.add_acquisition(ts2)
name = 'example_electrode_group'
desc = 'An example electrode'
loc = 'an example location'
device = self.nwbfile.create_device('a fake device')
elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, device)
children = self.nwbfile.all_children()
self.assertIn(ts1, children)
self.assertIn(ts2, children)
self.assertIn(device, children)
self.assertIn(elecgrp, children)
def test_fail_if_source_script_file_name_without_source_script(self):
with self.assertRaises(ValueError):
# <-- source_script_file_name without source_script is not allowed
NWBFile('a test session description for a test NWBFile',
'FILE123',
self.start,
source_script=None,
source_script_file_name='nofilename')
def test_get_neurodata_type(self):
ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
self.nwbfile.add_acquisition(ts1)
self.nwbfile.add_acquisition(ts2)
p1 = ts1.get_ancestor(neurodata_type='NWBFile')
self.assertIs(p1, self.nwbfile)
p2 = ts2.get_ancestor(neurodata_type='NWBFile')
self.assertIs(p2, self.nwbfile)
def test_print_units(self):
self.nwbfile.add_unit(spike_times=[1., 2., 3.])
expected = """units pynwb.misc.Units at 0x%d
Fields:
colnames: ['spike_times']
columns: (
spike_times_index <class 'hdmf.common.table.VectorIndex'>,
spike_times <class 'hdmf.common.table.VectorData'>
)
description: Autogenerated by NWBFile
id: id <class 'hdmf.common.table.ElementIdentifiers'>
waveform_unit: volts
"""
expected = expected % id(self.nwbfile.units)
self.assertEqual(str(self.nwbfile.units), expected)
def test_copy(self):
self.nwbfile.add_unit(spike_times=[1., 2., 3.])
device = self.nwbfile.create_device('a')
elecgrp = self.nwbfile.create_electrode_group('a',
'b',
device=device,
location='a')
self.nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=0)
self.nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'b', 'b',
elecgrp)
elec_region = self.nwbfile.create_electrode_table_region([1], 'name')
ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
ts2 = ElectricalSeries('test_ts2', [0, 1, 2, 3, 4, 5],
electrodes=elec_region,
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
self.nwbfile.add_acquisition(ts1)
self.nwbfile.add_acquisition(ts2)
self.nwbfile.add_trial(start_time=50.0, stop_time=70.0)
self.nwbfile.add_invalid_times_column(
'comments', 'description of reason for omitting time')
self.nwbfile.create_processing_module('test_mod', 'test_description')
self.nwbfile.create_time_intervals('custom_interval',
'a custom time interval')
self.nwbfile.intervals['custom_interval'].add_interval(start_time=10.,
stop_time=20.)
newfile = self.nwbfile.copy()
# test dictionaries
self.assertIs(self.nwbfile.devices['a'], newfile.devices['a'])
self.assertIs(self.nwbfile.acquisition['test_ts1'],
newfile.acquisition['test_ts1'])
self.assertIs(self.nwbfile.acquisition['test_ts2'],
newfile.acquisition['test_ts2'])
self.assertIs(self.nwbfile.processing['test_mod'],
newfile.processing['test_mod'])
# test dynamic tables
self.assertIsNot(self.nwbfile.electrodes, newfile.electrodes)
self.assertIs(self.nwbfile.electrodes['x'], newfile.electrodes['x'])
self.assertIsNot(self.nwbfile.units, newfile.units)
self.assertIs(self.nwbfile.units['spike_times'],
newfile.units['spike_times'])
self.assertIsNot(self.nwbfile.trials, newfile.trials)
self.assertIsNot(self.nwbfile.trials.parent, newfile.trials.parent)
self.assertIs(self.nwbfile.trials.id, newfile.trials.id)
self.assertIs(self.nwbfile.trials['start_time'],
newfile.trials['start_time'])
self.assertIs(self.nwbfile.trials['stop_time'],
newfile.trials['stop_time'])
self.assertIsNot(self.nwbfile.invalid_times, newfile.invalid_times)
self.assertTupleEqual(self.nwbfile.invalid_times.colnames,
newfile.invalid_times.colnames)
self.assertIsNot(self.nwbfile.intervals['custom_interval'],
newfile.intervals['custom_interval'])
self.assertTupleEqual(
self.nwbfile.intervals['custom_interval'].colnames,
newfile.intervals['custom_interval'].colnames)
self.assertIs(self.nwbfile.intervals['custom_interval']['start_time'],
newfile.intervals['custom_interval']['start_time'])
self.assertIs(self.nwbfile.intervals['custom_interval']['stop_time'],
newfile.intervals['custom_interval']['stop_time'])
def test_multi_experimenters(self):
self.nwbfile = NWBFile('a test session description for a test NWBFile',
'FILE123',
self.start,
experimenter=('experimenter1', 'experimenter2'))
self.assertTupleEqual(self.nwbfile.experimenter,
('experimenter1', 'experimenter2'))
def test_multi_publications(self):
self.nwbfile = NWBFile('a test session description for a test NWBFile',
'FILE123',
self.start,
related_publications=('pub1', 'pub2'))
self.assertTupleEqual(self.nwbfile.related_publications,
('pub1', 'pub2'))
def convert_from_old_neo(old_file, bird_name, electrode_df):
root_dir, fname = os.path.split(old_file)
rec_date = bird_info[bird_name]['recording_date']
rec_start = bird_info[bird_name]['recording_start']
rec_end = bird_info[bird_name]['recording_end']
rec_datetime = date_parser.parse('{} {}'.format(rec_date, rec_start))
# create an electrode array, electrode groups, and electrode table for the electrodes
i = electrode_df.bird == bird_name
edf = electrode_df[i]
for block, gdf in edf.groupby(['block']):
lfp_series = dict()
spike_series = dict()
print('*************** Processing block {}'.format(block))
if bird_name == 'WhiWhi4522M' and block == 'Site1':
continue
# get the LFP and spike data for the block
hf = h5py.File(old_file, 'r')
block_data = read_old_neo(hf, block)
hf.close()
recording_name = '{}_{}_{}'.format(bird_name, block,
block_data['protocol'])
nwb_file = os.path.join(root_dir, '{}.nwb'.format(recording_name))
session_desc = """ A single recording session, roughly one hour long, at a single depth, in a series of
recordings from the auditory "cortex" of zebra finch {}. Block {}, stimulus protocol
{}.
""".format(bird_name, block, block_data['protocol'])
exp_desc = """ A randomly interleaved mixture of Zebra finch vocalizations, songs, and modulation limited
noise played to a Zebra finch under urethane anaesthesia in an acute experiment. Two 16 microwire
electrode arrays were implanted, one in each hemisphere. Experiments designed and performed by
Julie Elie, vocal repertoire recorded by Julie Elie. Data converted to NWB by Mike Schachter. For
a full description of methods, please consult and also cite the following publications:
Elie, J. E., & Theunissen, F. E. (2015). Meaning in the avian auditory cortex: neural
representation of communication calls. European Journal of Neuroscience, 41(5), 546-567.
Elie, J. E., & Theunissen, F. E. (2016). The vocal repertoire of the domesticated zebra finch:
a data-driven approach to decipher the information-bearing acoustic features of communication
signals. Animal cognition, 19(2), 285-315.
"""
nf = NWBFile(recording_name,
session_desc,
bird_name,
rec_datetime,
experimenter='Julie Elie',
lab='Theunissen Lab',
institution='UC Berkeley',
experiment_description=exp_desc,
session_id=bird_name)
# create the electrodes and electrode tables
for hemi, ggdf in gdf.groupby(['hemisphere']):
electrode_array_name = '16 electrode microwire array on {} hemisphere'.format(
hemi)
electrode_array = nf.create_device(name=electrode_array_name,
source='')
# create an electrode group
egrp_desc = """ The (x,y) locations of the electrodes refer to the distance from region midline and distance
from L2A, respectively, in mm.
"""
electrode_group = nf.create_electrode_group(
hemi,
source=electrode_array_name,
description=egrp_desc,
location='{} Hemisphere, Field L, CM, NCM'.format(hemi),
device=electrode_array)
# add electrodes to electrode group
for row_idx, row in ggdf.iterrows():
electrode_number = row['electrode'] - 1
dist_l2a = row['dist_l2a']
dist_midline = row['dist_midline']
region = row['region']
if bird_name == 'GreBlu9508M':
dist_l2a *= 4 # correct for the error in the original data
nf.add_electrode(electrode_number,
x=dist_midline,
y=dist_l2a,
z=0.0,
imp=0.0,
location=region,
filtering='none',
description='Row {}, Column {}'.format(
row['row'], row['col']),
group=electrode_group)
# create an electrode table region
electrode_numbers = list(
np.array(sorted(ggdf.electrode.unique())) - 1)
print('electrode_numbers=', electrode_numbers)
etable = nf.create_electrode_table_region(
electrode_numbers,
'All electrodes in array for hemisphere {} with LFP'.format(
hemi))
lfp_data = np.array([
block_data['electrodes'][e + 1]['lfp']
for e in electrode_numbers
])
sr = block_data['sample_rate']
t = np.arange(lfp_data.shape[1]) / sr
# add the raw LFP
lfp_series_name = 'Multi-electrode LFP on {} hemisphere from block {}'.format(
hemi, block)
lfp = ElectricalSeries(
lfp_series_name,
electrode_array_name,
lfp_data,
etable,
timestamps=t,
resolution=1e-12,
comments='',
description='Low-passed LFP recorded from microwire array')
lfp_series[hemi] = lfp
# add the spikes and their waveforms
for row_idx, row in ggdf.iterrows():
# electrode_number ranges from 1-32, same as the keys in block_data['electrodes']
electrode_number = row['electrode']
for k, unit_data in enumerate(
block_data['electrodes'][electrode_number]['units']):
spike_times = unit_data['spike_times']
waveforms = unit_data['waveforms']
unit_name = unit_data['name']
e_num = unit_data['electrode']
print('{} electrode_number={}({}), e_num={}, k={}'.format(
unit_name, electrode_number, electrode_number - 1,
e_num, k))
assert e_num == electrode_number
if unit_name.startswith('RF Sort'):
xarr = unit_name.split(' ')
unit_num = xarr[-1]
sort_type = 'Spike-sorted unit {}'.format(unit_num)
elif unit_name.startswith('Chan'):
if 'Code0' not in unit_name:
print(
'Skipping multi-unit channel with non-zero sort code'
)
continue
sort_type = 'Multi-unit'
else:
raise Exception(
'Unknown unit name: {}'.format(unit_name))
full_unit_name = '{} on block {} and electrode {}'.format(
sort_type, block, e_num - 1)
spikes = SpikeEventSeries(
full_unit_name,
lfp_series_name,
waveforms,
spike_times,
etable,
resolution=1e-12,
conversion=1e6,
comments='',
description='',
)
print(
'\tAdding spikes acquisition: {} ({}), waveforms.shape={}'
.format(full_unit_name, sort_type,
str(waveforms.shape)))
spike_series[(hemi, electrode_number, unit_name)] = spikes
all_series = list()
all_series.extend(lfp_series.values())
all_series.extend(spike_series.values())
print('len(all_series)=', len(all_series))
nf.add_acquisition(all_series)
# create trials for each stim presentation
nf.add_trial_column('stim_id',
'The ID of the sound played during the trial.')
nf.add_trial_column(
'trial', 'The number of times this stimulus has been presented.')
for stim_epoch_data in block_data['epochs']:
stim_id = stim_epoch_data['stim_id']
trial_num = stim_epoch_data['trial']
stime = stim_epoch_data['start']
etime = stim_epoch_data['end']
nf.add_trial({
'start': stime,
'end': etime,
'stim_id': stim_id,
'trial': trial_num
})
print('Writing to {}'.format(nwb_file))
with NWBHDF5IO(nwb_file, mode='w') as io:
io.write(nf)
del nf
def save(self,
file_name,
to32=True,
order='F',
imagej=False,
bigtiff=True,
excitation_lambda=488.0,
compress=0,
var_name_hdf5='mov',
sess_desc='some_description',
identifier='some identifier',
imaging_plane_description='some imaging plane description',
emission_lambda=520.0,
indicator='OGB-1',
location='brain',
starting_time=0.,
experimenter='Dr Who',
lab_name=None,
institution=None,
experiment_description='Experiment Description',
session_id='Session ID'):
"""
Save the timeseries in single precision. Supported formats include
TIFF, NPZ, AVI, MAT, HDF5/H5, MMAP, and NWB
Args:
file_name: str
name of file. Possible formats are tif, avi, npz, mmap and hdf5
to32: Bool
whether to transform to 32 bits
order: 'F' or 'C'
C or Fortran order
var_name_hdf5: str
Name of hdf5 file subdirectory
Raises:
Exception 'Extension Unknown'
"""
name, extension = os.path.splitext(file_name)[:2]
extension = extension.lower()
logging.debug("Parsing extension " + str(extension))
if extension == '.tif':
with tifffile.TiffWriter(file_name, bigtiff=bigtiff,
imagej=imagej) as tif:
for i in range(self.shape[0]):
if i % 200 == 0:
logging.debug(str(i) + ' frames saved')
curfr = self[i].copy()
if to32 and not ('float32' in str(self.dtype)):
curfr = curfr.astype(np.float32)
tif.save(curfr, compress=compress)
elif extension == '.npz':
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
np.savez(file_name,
input_arr=input_arr,
start_time=self.start_time,
fr=self.fr,
meta_data=self.meta_data,
file_name=self.file_name)
elif extension == '.avi':
codec = None
try:
codec = cv2.FOURCC('I', 'Y', 'U', 'V')
except AttributeError:
codec = cv2.VideoWriter_fourcc(*'IYUV')
np.clip(self, np.percentile(self, 1), np.percentile(self, 99),
self)
minn, maxx = np.min(self), np.max(self)
data = 255 * (self - minn) / (maxx - minn)
data = data.astype(np.uint8)
y, x = data[0].shape
vw = cv2.VideoWriter(file_name,
codec,
self.fr, (x, y),
isColor=True)
for d in data:
vw.write(cv2.cvtColor(d, cv2.COLOR_GRAY2BGR))
vw.release()
elif extension == '.mat':
if self.file_name[0] is not None:
f_name = self.file_name
else:
f_name = ''
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
if self.meta_data[0] is None:
savemat(
file_name, {
'input_arr': np.rollaxis(input_arr, axis=0, start=3),
'start_time': self.start_time,
'fr': self.fr,
'meta_data': [],
'file_name': f_name
})
else:
savemat(
file_name, {
'input_arr': np.rollaxis(input_arr, axis=0, start=3),
'start_time': self.start_time,
'fr': self.fr,
'meta_data': self.meta_data,
'file_name': f_name
})
elif extension in ('.hdf5', '.h5'):
with h5py.File(file_name, "w") as f:
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
dset = f.create_dataset(var_name_hdf5, data=input_arr)
dset.attrs["fr"] = self.fr
dset.attrs["start_time"] = self.start_time
try:
dset.attrs["file_name"] = [
a.encode('utf8') for a in self.file_name
]
except:
logging.warning('No file saved')
if self.meta_data[0] is not None:
logging.debug("Metadata for saved file: " +
str(self.meta_data))
dset.attrs["meta_data"] = cpk.dumps(self.meta_data)
elif extension == '.mmap':
base_name = name
T = self.shape[0]
dims = self.shape[1:]
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
input_arr = np.transpose(input_arr,
list(range(1,
len(dims) + 1)) + [0])
input_arr = np.reshape(input_arr, (np.prod(dims), T), order='F')
fname_tot = memmap_frames_filename(base_name, dims, T, order)
fname_tot = os.path.join(os.path.split(file_name)[0], fname_tot)
big_mov = np.memmap(fname_tot,
mode='w+',
dtype=np.float32,
shape=(np.uint64(np.prod(dims)), np.uint64(T)),
order=order)
big_mov[:] = np.asarray(input_arr, dtype=np.float32)
big_mov.flush()
del big_mov, input_arr
return fname_tot
elif extension == '.nwb':
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
# Create NWB file
nwbfile = NWBFile(sess_desc,
identifier,
datetime.now(tzlocal()),
experimenter=experimenter,
lab=lab_name,
institution=institution,
experiment_description=experiment_description,
session_id=session_id)
# Get the device
device = Device('imaging_device')
nwbfile.add_device(device)
# OpticalChannel
optical_channel = OpticalChannel('OpticalChannel',
'main optical channel',
emission_lambda=emission_lambda)
imaging_plane = nwbfile.create_imaging_plane(
name='ImagingPlane',
optical_channel=optical_channel,
description=imaging_plane_description,
device=device,
excitation_lambda=excitation_lambda,
imaging_rate=self.fr,
indicator=indicator,
location=location)
# Images
image_series = TwoPhotonSeries(name=var_name_hdf5,
dimension=self.shape[1:],
data=input_arr,
imaging_plane=imaging_plane,
starting_frame=[0],
starting_time=starting_time,
rate=self.fr)
nwbfile.add_acquisition(image_series)
with NWBHDF5IO(file_name, 'w') as io:
io.write(nwbfile)
return file_name
else:
logging.error("Extension " + str(extension) + " unknown")
raise Exception('Extension Unknown')
class TestNWBFileHDF5IO(TestCase):
""" Test reading/writing an NWBFile using HDF5IO """
def setUp(self):
""" Set up an NWBFile object with an acquisition TimeSeries, analysis TimeSeries, and a processing module """
self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc())
self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc())
self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal())
self.manager = get_manager()
self.filename = 'test_nwbfileio.h5'
self.nwbfile = NWBFile(
session_description='a test NWB File',
identifier='TEST123',
session_start_time=self.start_time,
timestamps_reference_time=self.ref_time,
file_create_date=self.create_date,
experimenter='test experimenter',
stimulus_notes='test stimulus notes',
data_collection='test data collection notes',
experiment_description='test experiment description',
institution='nomad',
lab='nolab',
notes='nonotes',
pharmacology='nopharmacology',
protocol='noprotocol',
related_publications='nopubs',
session_id='007',
slices='noslices',
source_script='nosources',
surgery='nosurgery',
virus='novirus',
source_script_file_name='nofilename')
self.ts = TimeSeries(name='test_timeseries',
data=list(range(100, 200, 10)),
unit='SIunit',
timestamps=np.arange(10.),
resolution=0.1)
self.nwbfile.add_acquisition(self.ts)
self.ts2 = TimeSeries(name='test_timeseries2',
data=list(range(200, 300, 10)),
unit='SIunit',
timestamps=np.arange(10.),
resolution=0.1)
self.nwbfile.add_analysis(self.ts2)
self.mod = self.nwbfile.create_processing_module(
'test_module', 'a test module')
self.ts3 = TimeSeries(name='test_timeseries2',
data=list(range(100, 200, 10)),
unit='SIunit',
timestamps=np.arange(10.),
resolution=0.1)
self.mod.add(self.ts3)
def tearDown(self):
""" Delete the created test file """
remove_test_file(self.filename)
def test_children(self):
""" Test that the TimeSeries and processing module are children of their respective parents """
self.assertIn(self.ts, self.nwbfile.children)
self.assertIn(self.ts2, self.nwbfile.children)
self.assertIn(self.mod, self.nwbfile.children)
self.assertIn(self.ts3, self.mod.children)
def test_write(self):
""" Test writing the NWBFile using HDF5IO """
hdf5io = HDF5IO(self.filename, manager=self.manager, mode='a')
hdf5io.write(self.nwbfile)
hdf5io.close()
# TODO add some asserts
def test_read(self):
""" Test reading the NWBFile using HDF5IO """
hdf5io = HDF5IO(self.filename, manager=self.manager, mode='w')
hdf5io.write(self.nwbfile)
hdf5io.close()
hdf5io = HDF5IO(self.filename, manager=self.manager, mode='r')
container = hdf5io.read()
self.assertIsInstance(container, NWBFile)
self.assertEqual(len(container.acquisition), 1)
self.assertEqual(len(container.analysis), 1)
for v in container.acquisition.values():
self.assertIsInstance(v, TimeSeries)
self.assertContainerEqual(container, self.nwbfile)
hdf5io.close()
unit='m',
starting_time=0.0,
rate=1.0)
####################
# Using this scheme says that this :py:class:`~pynwb.base.TimeSeries` started recording 0 seconds after
# *start_time* stored in the :py:class:`~pynwb.file.NWBFile` and sampled every second.
#
# :py:class:`~pynwb.base.TimeSeries` objects can be added directly to your :py:class:`~pynwb.file.NWBFile` using
# the methods :py:meth:`~pynwb.file.NWBFile.add_acquisition`, :py:meth:`~pynwb.file.NWBFile.add_stimulus`
# and :py:meth:`~pynwb.file.NWBFile.add_stimulus_template`. Which method you use depends on the source of the
# data: use :py:meth:`~pynwb.file.NWBFile.add_acquisition` to indicated *acquisition* data,
# :py:meth:`~pynwb.file.NWBFile.add_stimulus` to indicate *stimulus* data, and
# :py:meth:`~pynwb.file.NWBFile.add_stimulus_template` to store stimulus templates.
nwbfile.add_acquisition(test_ts)
####################
# Access the :py:class:`~pynwb.base.TimeSeries` object `'test_timeseries'` from *acquisition* using
nwbfile.acquisition['test_timeseries']
####################
# or
nwbfile.get_acquisition('test_timeseries')
####################
# .. _basic_writing:
#
# Writing an NWB file
# -------------------
#
from pynwb.image import ImageSeries
from pynwb import NWBHDF5IO, NWBFile
from h5py import File
import numpy as np
from datetime import datetime
base_dir = '/Users/bendichter/Desktop/Schnitzer/data/Example Data'
nwbfile = NWBFile(session_start_time=datetime(1900, 1, 1),
session_description=' ',
identifier='m655_D11_S1')
nwbfile.add_acquisition(
ImageSeries(name='video',
format='external',
external_file=['m655_D11_S1.avi'],
starting_time=0.0,
rate=np.nan,
starting_frame=[0],
dimension=[250, 250]))
centroid_fname = 'm655_D11_S1_centroids.mat'
pos_path = os.path.join(base_dir, centroid_fname)
with File(pos_path, 'r') as file:
pos_data = np.array(file['c']).T
spatial_series = SpatialSeries(name='position',
data=pos_data,
starting_time=0.0,
rate=5.0,
units='unknown',
reference_frame='unknown')
class TestHDF5Writer(unittest.TestCase):
_required_tests = ('test_nwbio', 'test_write_clobber',
'test_write_cache_spec')
@property
def required_tests(self):
return self._required_tests
def setUp(self):
self.manager = get_manager()
self.path = "test_pynwb_io_hdf5.h5"
self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc())
self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal())
self.container = NWBFile('a test NWB File',
'TEST123',
self.start_time,
file_create_date=self.create_date)
ts = TimeSeries('test_timeseries',
list(range(100, 200, 10)),
'SIunit',
timestamps=list(range(10)),
resolution=0.1)
self.container.add_acquisition(ts)
ts_builder = GroupBuilder('test_timeseries',
attributes={
'neurodata_type': 'TimeSeries',
'help': 'General purpose TimeSeries'
},
datasets={
'data':
DatasetBuilder('data',
list(range(100, 200, 10)),
attributes={
'unit': 'SIunit',
'conversion': 1.0,
'resolution': 0.1
}),
'timestamps':
DatasetBuilder('timestamps',
list(range(10)),
attributes={
'unit': 'Seconds',
'interval': 1
})
})
self.builder = GroupBuilder(
'root',
groups={
'acquisition':
GroupBuilder('acquisition',
groups={'test_timeseries': ts_builder}),
'analysis':
GroupBuilder('analysis'),
'general':
GroupBuilder('general'),
'processing':
GroupBuilder('processing'),
'stimulus':
GroupBuilder('stimulus',
groups={
'presentation': GroupBuilder('presentation'),
'templates': GroupBuilder('templates')
})
},
datasets={
'file_create_date':
DatasetBuilder('file_create_date',
[self.create_date.isoformat()]),
'identifier':
DatasetBuilder('identifier', 'TEST123'),
'session_description':
DatasetBuilder('session_description', 'a test NWB File'),
'nwb_version':
DatasetBuilder('nwb_version', '1.0.6'),
'session_start_time':
DatasetBuilder('session_start_time',
self.start_time.isoformat())
},
attributes={'neurodata_type': 'NWBFile'})
def tearDown(self):
os.remove(self.path)
def test_nwbio(self):
io = HDF5IO(self.path, manager=self.manager, mode='a')
io.write(self.container)
io.close()
f = File(self.path)
self.assertIn('acquisition', f)
self.assertIn('analysis', f)
self.assertIn('general', f)
self.assertIn('processing', f)
self.assertIn('file_create_date', f)
self.assertIn('identifier', f)
self.assertIn('session_description', f)
self.assertIn('session_start_time', f)
acq = f.get('acquisition')
self.assertIn('test_timeseries', acq)
def test_write_clobber(self):
io = HDF5IO(self.path, manager=self.manager, mode='a')
io.write(self.container)
io.close()
f = File(self.path) # noqa: F841
if six.PY2:
assert_file_exists = IOError
elif six.PY3:
assert_file_exists = OSError
with self.assertRaises(assert_file_exists):
io = HDF5IO(self.path, manager=self.manager, mode='w-')
io.write(self.container)
io.close()
def test_write_cache_spec(self):
'''
Round-trip test for writing spec and reading it back in
'''
io = HDF5IO(self.path, manager=self.manager, mode="a")
io.write(self.container, cache_spec=True)
io.close()
f = File(self.path)
self.assertIn('specifications', f)
ns_catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec,
NWBNamespace)
HDF5IO.load_namespaces(ns_catalog, self.path, namespaces=['core'])
original_ns = self.manager.namespace_catalog.get_namespace('core')
cached_ns = ns_catalog.get_namespace('core')
self.maxDiff = None
for key in ('author', 'contact', 'doc', 'full_name', 'name'):
with self.subTest(namespace_field=key):
self.assertEqual(original_ns[key], cached_ns[key])
for dt in original_ns.get_registered_types():
with self.subTest(neurodata_type=dt):
original_spec = original_ns.get_spec(dt)
cached_spec = cached_ns.get_spec(dt)
with self.subTest(test='data_type spec read back in'):
self.assertIsNotNone(cached_spec)
with self.subTest(test='cached spec preserved original spec'):
self.assertDictEqual(original_spec, cached_spec)
deconv_roi_response_series = DeconvolvedRoiResponseSeries(
name="deconvolved_fluorescence_trace",
description="my roi response series",
data=np.random.randn(100, 1),
unit='F',
rate=30.0,
rois=fibers_ref,
raw=roi_response_series,
)
ophys_module = nwbfile.create_processing_module(
name="ophys", description="fiber photometry"
)
ophys_module.add(multi_commanded_voltage)
nwbfile.add_acquisition(roi_response_series)
ophys_module.add(deconv_roi_response_series)
# write nwb file
filename = 'test.nwb'
with NWBHDF5IO(filename, 'w') as io:
io.write(nwbfile)
# read nwb file and check its contents
with NWBHDF5IO(filename, 'r', load_namespaces=True) as io:
nwbfile = io.read()
# Access and print information about the acquisition
print(nwbfile.acquisition["raw_fluorescence_trace"])
# Access and print information about the processed data
print(nwbfile.processing['ophys']["deconvolved_fluorescence_trace"])
# Access and print all of the metadata
def run_conversion(self, nwbfile: NWBFile, metadata: dict, stub_test: bool = False):
super().run_conversion(nwbfile=nwbfile, metadata=metadata, stub_test=stub_test)
session_path = Path(self.source_data["file_path"]).parent
session_id = session_path.name
subject_path = session_path.parent
xml_filepath = session_path / f"{session_id}.xml"
root = et.parse(str(xml_filepath)).getroot()
n_total_channels = int(root.find("acquisitionSystem").find("nChannels").text)
lfp_sampling_rate = float(root.find("fieldPotentials").find("lfpSamplingRate").text)
shank_channels = [
[int(channel.text) for channel in group.find("channels")]
for group in root.find("spikeDetection").find("channelGroups").findall("group")
]
all_shank_channels = np.concatenate(shank_channels) # Flattened
# Special electrodes
special_electrode_mapping = dict(
ch_wait=79,
ch_arm=78,
ch_solL=76,
ch_solR=77,
ch_dig1=65,
ch_dig2=68,
ch_entL=72,
ch_entR=71,
ch_SsolL=73,
ch_SsolR=70,
)
special_electrodes = []
for special_electrode_name, channel in special_electrode_mapping.items():
if channel <= n_total_channels - 1:
special_electrodes.append(
dict(
name=special_electrode_name,
channel=channel,
description="Environmental electrode recorded inline with neural data.",
)
)
_, all_channels_lfp_data = read_lfp(session_path, stub=stub_test)
for special_electrode in special_electrodes:
ts = TimeSeries(
name=special_electrode["name"],
description=special_electrode["description"],
data=all_channels_lfp_data[:, special_electrode["channel"]],
rate=lfp_sampling_rate,
unit="V",
resolution=np.nan,
)
nwbfile.add_acquisition(ts)
# DecompositionSeries
mouse_number = session_id[-9:-7]
subject_xls = str(subject_path / f"DGProject/YM{mouse_number} exp_sheet.xlsx")
hilus_csv_path = str(subject_path / "DGProject/early_session_hilus_chans.csv")
session_start = metadata["NWBFile"]["session_start_time"]
if "-" in session_id:
b = False
else:
b = True
lfp_channel = get_reference_elec(subject_xls, hilus_csv_path, session_start, session_id, b=b)
if lfp_channel is not None:
lfp_data = all_channels_lfp_data[:, all_shank_channels]
all_lfp_phases = []
for passband in ("theta", "gamma"):
lfp_fft = filter_lfp(
lfp_data[:, all_shank_channels == lfp_channel].ravel(),
lfp_sampling_rate,
passband=passband,
)
lfp_phase, _ = hilbert_lfp(lfp_fft)
all_lfp_phases.append(lfp_phase[:, np.newaxis])
decomp_series_data = np.dstack(all_lfp_phases)
ecephys_mod = check_module(
nwbfile,
"ecephys",
"Intermediate data from extracellular electrophysiology recordings, e.g., LFP.",
)
lfp_ts = ecephys_mod.data_interfaces["LFP"]["LFP"]
decomp_series = DecompositionSeries(
name="LFPDecompositionSeries",
description="Theta and Gamma phase for reference LFP",
data=decomp_series_data,
rate=lfp_sampling_rate,
source_timeseries=lfp_ts,
metric="phase",
unit="radians",
)
# TODO: the band limits should be extracted from parse_passband in band_analysis?
decomp_series.add_band(band_name="theta", band_limits=(4, 10))
decomp_series.add_band(band_name="gamma", band_limits=(30, 80))
check_module(
nwbfile,
"ecephys",
"Contains processed extracellular electrophysiology data.",
).add(decomp_series)
session_description='demonstrate NWBFile scratch', # required
identifier='NWB456', # required
session_start_time=start_time, # required
file_create_date=create_date) # optional
# make some fake data
timestamps = np.linspace(0, 100, 1024)
data = np.sin(0.333 * timestamps) + np.cos(0.1 * timestamps) + np.random.randn(
len(timestamps))
test_ts = TimeSeries(name='raw_timeseries',
data=data,
unit='m',
timestamps=timestamps)
# add it to the NWBFile
nwb.add_acquisition(test_ts)
with NWBHDF5IO('raw_data.nwb', mode='w') as io:
io.write(nwb)
####################
# .. _basic_copying:
#
# Copying an NWB file
# -------------------
#
# To copy a file, we must first read the file.
#
raw_io = NWBHDF5IO('raw_data.nwb', 'r')
nwb_in = raw_io.read()
group=electrode_group)
ephys_table_region = nwbfile.create_electrode_table_region(list(range(5)), 'all ephys electrodes')
ophys_device = nwbfile.create_device('ophys_device', source=source)
ogen_site = nwbfile.create_ogen_site('oPhys', ophys_device,
description='unknown',
excitation_lambda='unknown',
location='unknown')
module = nwbfile.create_processing_module(name='0', description=source)
for i, trial_data in enumerate(data):
nwbfile.add_acquisition(
ElectricalSeries('ePhys trial' + str(i),
gzip(trial_data.ephys[:, [0, 1, 2, 6, 7]]),
ephys_table_region,
timestamps=trial_data.time)
)
nwbfile.add_acquisition(
OptogeneticSeries('oPhys trial' + str(i),
gzip(trial_data.tempo_data[:, [0, 6, 7]]),
ogen_site,
description='laser, reference, voltage',
timestamps=trial_data.time))
with NWBHDF5IO('/Users/bendichter/Desktop/Schnitzer/data/simon_out.nwb', 'w') as io:
io.write(nwbfile)
#trial_data.ephys
def chang2nwb(blockpath, out_file_path=None, save_to_file=False, htk_config=None):
"""
Parameters
----------
blockpath: str
out_file_path: None | str
if None, output = [blockpath]/[blockname].nwb
save_to_file : bool
If True, saves to file. If False, just returns nwbfile object
htk_config : dict
Dictionary cotaining HTK conversion paths and options. Example:
{
ecephys_path: 'path_to/ecephys_htk_files',
ecephys_type: 'raw', 'preprocessed' or 'high_gamma',
analog_path: 'path_to/analog_htk_files',
anin1: {present: True, name: 'microphone', type: 'acquisition'},
anin2: {present: True, name: 'speaker1', type: 'stimulus'},
anin3: {present: False, name: 'speaker2', type: 'stimulus'},
anin4: {present: False, name: 'custom', type: 'acquisition'},
metadata: metadata,
electrodes_file: electrodes_file,
bipolar_file: bipolar_file
}
Returns
-------
"""
metadata = {}
if htk_config is None:
blockpath = Path(blockpath)
else:
blockpath = Path(htk_config['ecephys_path'])
metadata = htk_config['metadata']
blockname = blockpath.parent.name
subject_id = blockpath.parent.parent.name[2:]
if out_file_path is None:
out_file_path = blockpath.resolve().parent / ''.join(['EC', subject_id, '_', blockname, '.nwb'])
# file paths
ecog_path = blockpath
anin_path = htk_config['analog_path']
bad_time_file = path.join(blockpath, 'Artifacts', 'badTimeSegments.mat')
# Create the NWB file object
nwbfile_dict = {
'session_description': blockname,
'identifier': blockname,
'session_start_time': datetime.now().astimezone(),
'institution': 'University of California, San Francisco',
'lab': 'Chang Lab'
}
if 'NWBFile' in metadata:
nwbfile_dict.update(metadata['NWBFile'])
nwbfile = NWBFile(**nwbfile_dict)
# Read electrophysiology data from HTK files
print('reading htk acquisition...', flush=True)
ecog_rate, data = readhtks(ecog_path)
data = data.squeeze()
print('done', flush=True)
# Get electrodes info from mat file
if htk_config['electrodes_file'] is not None:
nwbfile = elecs_to_electrode_table(
nwbfile=nwbfile,
elecspath=htk_config['electrodes_file'],
)
n_electrodes = nwbfile.electrodes[:].shape[0]
all_elecs = list(range(n_electrodes))
elecs_region = nwbfile.create_electrode_table_region(
region=all_elecs,
description='ECoG electrodes on brain'
)
else:
ecephys_dict = {
'Device': [{'name': 'auto_device'}],
'ElectricalSeries': [{'name': 'ECoG', 'description': 'description'}],
'ElectrodeGroup': [{'name': 'auto_group', 'description': 'auto_group',
'location': 'location', 'device': 'auto_device'}]
}
if 'Ecephys' in metadata:
ecephys_dict.update(metadata['Ecephys'])
# Create devices
for dev in ecephys_dict['Device']:
device = nwbfile.create_device(dev['name'])
# Electrode groups
for el_grp in ecephys_dict['ElectrodeGroup']:
device = nwbfile.devices[el_grp['device']]
electrode_group = nwbfile.create_electrode_group(
name=el_grp['name'],
description=el_grp['description'],
location=el_grp['location'],
device=device
)
# Electrodes table
n_electrodes = data.shape[1]
nwbfile.add_electrode_column('label', 'label of electrode')
nwbfile.add_electrode_column('bad', 'electrode identified as too noisy')
nwbfile.add_electrode_column('x_warped', 'x warped onto cvs_avg35_inMNI152')
nwbfile.add_electrode_column('y_warped', 'y warped onto cvs_avg35_inMNI152')
nwbfile.add_electrode_column('z_warped', 'z warped onto cvs_avg35_inMNI152')
nwbfile.add_electrode_column('null', 'if not connected to real electrode')
bad_elecs_inds = get_bad_elecs(blockpath)
for elec_counter in range(n_electrodes):
bad = elec_counter in bad_elecs_inds
nwbfile.add_electrode(
id=elec_counter,
x=np.nan,
y=np.nan,
z=np.nan,
imp=np.nan,
x_warped=np.nan,
y_warped=np.nan,
z_warped=np.nan,
location='',
filtering='none',
group=electrode_group,
label='',
bad=bad,
null=False,
)
all_elecs = list(range(n_electrodes))
elecs_region = nwbfile.create_electrode_table_region(
region=all_elecs,
description='ECoG electrodes on brain'
)
# Get Bipolar table from file
if htk_config['bipolar_file'] is not None:
df = pd.read_csv(htk_config['bipolar_file'], index_col='id', sep='\t')
# Create bipolar scheme table
bipolar_scheme_table = BipolarSchemeTable(
name='bipolar_scheme_table',
description='desc'
)
# Columns for bipolar scheme - all anodes and cathodes within the same
# bipolar row are considered to have the same group and location
bipolar_scheme_table.add_column(
name='group_name',
description='electrode group name'
)
bipolar_scheme_table.add_column(
name='location',
description='electrode location'
)
# Iterate over anode / cathode rows
for i, r in df.iterrows():
if isinstance(r['anodes'], str):
anodes = [int(a) for a in r['anodes'].split(',')]
else:
anodes = [int(r['anodes'])]
if isinstance(r['cathodes'], str):
cathodes = [int(a) for a in r['cathodes'].split(',')]
else:
cathodes = [int(r['cathodes'])]
bipolar_scheme_table.add_row(
anodes=anodes,
cathodes=cathodes,
group_name=nwbfile.electrodes['group_name'][anodes[0]],
location=nwbfile.electrodes['location'][anodes[0]]
)
bipolar_scheme_table.anodes.table = nwbfile.electrodes
bipolar_scheme_table.cathodes.table = nwbfile.electrodes
# Creates bipolar table region
elecs_region = DynamicTableRegion(
name='electrodes',
data=np.arange(0, df.shape[0]),
description='desc',
table=bipolar_scheme_table
)
ecephys_ext = EcephysExt(name='ecephys_ext')
ecephys_ext.bipolar_scheme_table = bipolar_scheme_table
nwbfile.add_lab_meta_data(ecephys_ext)
# Stores HTK electrophysiology data as raw, preprocessed or high gamma
if htk_config['ecephys_type'] == 'raw':
ecog_es = ElectricalSeries(name='ECoG',
data=H5DataIO(data[:, 0:n_electrodes], compression='gzip'),
electrodes=elecs_region,
rate=ecog_rate,
description='all Wav data')
nwbfile.add_acquisition(ecog_es)
elif htk_config['ecephys_type'] == 'preprocessed':
lfp = LFP()
ecog_es = ElectricalSeries(name='preprocessed',
data=H5DataIO(data[:, 0:n_electrodes], compression='gzip'),
electrodes=elecs_region,
rate=ecog_rate,
description='all Wav data')
lfp.add_electrical_series(ecog_es)
# Creates the ecephys processing module
ecephys_module = nwbfile.create_processing_module(
name='ecephys',
description='preprocessed electrophysiology data'
)
ecephys_module.add_data_interface(lfp)
elif htk_config['ecephys_type'] == 'high_gamma':
ecog_es = ElectricalSeries(name='high_gamma',
data=H5DataIO(data[:, 0:n_electrodes], compression='gzip'),
electrodes=elecs_region,
rate=ecog_rate,
description='all Wav data')
# Creates the ecephys processing module
ecephys_module = nwbfile.create_processing_module(
name='ecephys',
description='preprocessed electrophysiology data'
)
ecephys_module.add_data_interface(ecog_es)
# Add ANIN 1
if htk_config['anin1']['present']:
fs, data = get_analog(anin_path, 1)
ts = TimeSeries(
name=htk_config['anin1']['name'],
data=data,
unit='NA',
rate=fs,
)
if htk_config['anin1']['type'] == 'acquisition':
nwbfile.add_acquisition(ts)
else:
nwbfile.add_stimulus(ts)
print('ANIN1 saved with name "', htk_config['anin1']['name'], '" in ',
htk_config['anin1']['type'])
# Add ANIN 2
if htk_config['anin2']['present']:
fs, data = get_analog(anin_path, 2)
ts = TimeSeries(
name=htk_config['anin2']['name'],
data=data,
unit='NA',
rate=fs,
)
if htk_config['anin2']['type'] == 'acquisition':
nwbfile.add_acquisition(ts)
else:
nwbfile.add_stimulus(ts)
print('ANIN2 saved with name "', htk_config['anin2']['name'], '" in ',
htk_config['anin2']['type'])
# Add ANIN 3
if htk_config['anin3']['present']:
fs, data = get_analog(anin_path, 3)
ts = TimeSeries(
name=htk_config['anin3']['name'],
data=data,
unit='NA',
rate=fs,
)
if htk_config['anin3']['type'] == 'acquisition':
nwbfile.add_acquisition(ts)
else:
nwbfile.add_stimulus(ts)
print('ANIN3 saved with name "', htk_config['anin3']['name'], '" in ',
htk_config['anin3']['type'])
# Add ANIN 4
if htk_config['anin4']['present']:
fs, data = get_analog(anin_path, 4)
ts = TimeSeries(
name=htk_config['anin4']['name'],
data=data,
unit='NA',
rate=fs,
)
if htk_config['anin4']['type'] == 'acquisition':
nwbfile.add_acquisition(ts)
else:
nwbfile.add_stimulus(ts)
print('ANIN4 saved with name "', htk_config['anin4']['name'], '" in ',
htk_config['anin4']['type'])
# Add bad time segments
if os.path.exists(bad_time_file) and os.stat(bad_time_file).st_size:
bad_time = sio.loadmat(bad_time_file)['badTimeSegments']
for row in bad_time:
nwbfile.add_invalid_time_interval(start_time=row[0],
stop_time=row[1],
tags=('ECoG artifact',),
timeseries=ecog_es)
# Subject
subject_dict = {'subject_id': subject_id}
if 'Subject' in metadata:
subject_dict.update(metadata['Subject'])
subject = ECoGSubject(**subject_dict)
nwbfile.subject = subject
if save_to_file:
print('Saving HTK content to NWB file...')
# Export the NWB file
with NWBHDF5IO(str(out_file_path), manager=manager, mode='w') as io:
io.write(nwbfile)
# read check
with NWBHDF5IO(str(out_file_path), manager=manager, mode='r') as io:
io.read()
print('NWB file saved: ', str(out_file_path))
return nwbfile, out_file_path, subject_id, blockname
def convert(
input_file,
session_start_time,
subject_date_of_birth,
subject_id='I5',
subject_description='naive',
subject_genotype='wild-type',
subject_sex='M',
subject_weight='11.6g',
subject_species='Mus musculus',
subject_brain_region='Medial Entorhinal Cortex',
surgery='Probe: +/-3.3mm ML, 0.2mm A of sinus, then as deep as possible',
session_id='npI5_0417_baseline_1',
experimenter='Kei Masuda',
experiment_description='Virtual Hallway Task',
institution='Stanford University School of Medicine',
lab_name='Giocomo Lab'):
"""
Read in the .mat file specified by input_file and convert to .nwb format.
Parameters
----------
input_file : np.ndarray (..., n_channels, n_time)
the .mat file to be converted
subject_id : string
the unique subject ID number for the subject of the experiment
subject_date_of_birth : datetime ISO 8601
the date and time the subject was born
subject_description : string
important information specific to this subject that differentiates it from other members of it's species
subject_genotype : string
the genetic strain of this species.
subject_sex : string
Male or Female
subject_weight :
the weight of the subject around the time of the experiment
subject_species : string
the name of the species of the subject
subject_brain_region : basestring
the name of the brain region where the electrode probe is recording from
surgery : str
information about the subject's surgery to implant electrodes
session_id: string
human-readable ID# for the experiment session that has a one-to-one relationship with a recording session
session_start_time : datetime
date and time that the experiment started
experimenter : string
who ran the experiment, first and last name
experiment_description : string
what task was being run during the session
institution : string
what institution was the experiment performed in
lab_name : string
the lab where the experiment was performed
Returns
-------
nwbfile : NWBFile
The contents of the .mat file converted into the NWB format. The nwbfile is saved to disk using NDWHDF5
"""
# input matlab data
matfile = hdf5storage.loadmat(input_file)
# output path for nwb data
def replace_last(source_string, replace_what, replace_with):
head, _sep, tail = source_string.rpartition(replace_what)
return head + replace_with + tail
outpath = replace_last(input_file, '.mat', '.nwb')
create_date = datetime.today()
timezone_cali = pytz.timezone('US/Pacific')
create_date_tz = timezone_cali.localize(create_date)
# if loading data from config.yaml, convert string dates into datetime
if isinstance(session_start_time, str):
session_start_time = datetime.strptime(session_start_time,
'%B %d, %Y %I:%M%p')
session_start_time = timezone_cali.localize(session_start_time)
if isinstance(subject_date_of_birth, str):
subject_date_of_birth = datetime.strptime(subject_date_of_birth,
'%B %d, %Y %I:%M%p')
subject_date_of_birth = timezone_cali.localize(subject_date_of_birth)
# create unique identifier for this experimental session
uuid_identifier = uuid.uuid1()
# Create NWB file
nwbfile = NWBFile(
session_description=experiment_description, # required
identifier=uuid_identifier.hex, # required
session_id=session_id,
experiment_description=experiment_description,
experimenter=experimenter,
surgery=surgery,
institution=institution,
lab=lab_name,
session_start_time=session_start_time, # required
file_create_date=create_date_tz) # optional
# add information about the subject of the experiment
experiment_subject = Subject(subject_id=subject_id,
species=subject_species,
description=subject_description,
genotype=subject_genotype,
date_of_birth=subject_date_of_birth,
weight=subject_weight,
sex=subject_sex)
nwbfile.subject = experiment_subject
# adding constants via LabMetaData container
# constants
sample_rate = float(matfile['sp'][0]['sample_rate'][0][0][0])
n_channels_dat = int(matfile['sp'][0]['n_channels_dat'][0][0][0])
dat_path = matfile['sp'][0]['dat_path'][0][0][0]
offset = int(matfile['sp'][0]['offset'][0][0][0])
data_dtype = matfile['sp'][0]['dtype'][0][0][0]
hp_filtered = bool(matfile['sp'][0]['hp_filtered'][0][0][0])
vr_session_offset = matfile['sp'][0]['vr_session_offset'][0][0][0]
# container
lab_metadata = LabMetaData_ext(name='LabMetaData',
acquisition_sampling_rate=sample_rate,
number_of_electrodes=n_channels_dat,
file_path=dat_path,
bytes_to_skip=offset,
raw_data_dtype=data_dtype,
high_pass_filtered=hp_filtered,
movie_start_time=vr_session_offset)
nwbfile.add_lab_meta_data(lab_metadata)
# Adding trial information
nwbfile.add_trial_column(
'trial_contrast',
'visual contrast of the maze through which the mouse is running')
trial = np.ravel(matfile['trial'])
trial_nums = np.unique(trial)
position_time = np.ravel(matfile['post'])
# matlab trial numbers start at 1. To correctly index trial_contract vector,
# subtracting 1 from 'num' so index starts at 0
for num in trial_nums:
trial_times = position_time[trial == num]
nwbfile.add_trial(start_time=trial_times[0],
stop_time=trial_times[-1],
trial_contrast=matfile['trial_contrast'][num - 1][0])
# Add mouse position inside:
position = Position()
position_virtual = np.ravel(matfile['posx'])
# position inside the virtual environment
sampling_rate = 1 / (position_time[1] - position_time[0])
position.create_spatial_series(
name='Position',
data=position_virtual,
starting_time=position_time[0],
rate=sampling_rate,
reference_frame='The start of the trial, which begins at the start '
'of the virtual hallway.',
conversion=0.01,
description='Subject position in the virtual hallway.',
comments='The values should be >0 and <400cm. Values greater than '
'400cm mean that the mouse briefly exited the maze.',
)
# physical position on the mouse wheel
physical_posx = position_virtual
trial_gain = np.ravel(matfile['trial_gain'])
for num in trial_nums:
physical_posx[trial ==
num] = physical_posx[trial == num] / trial_gain[num - 1]
position.create_spatial_series(
name='PhysicalPosition',
data=physical_posx,
starting_time=position_time[0],
rate=sampling_rate,
reference_frame='Location on wheel re-referenced to zero '
'at the start of each trial.',
conversion=0.01,
description='Physical location on the wheel measured '
'since the beginning of the trial.',
comments='Physical location found by dividing the '
'virtual position by the "trial_gain"')
nwbfile.add_acquisition(position)
# Add timing of lick events, as well as mouse's virtual position during lick event
lick_events = BehavioralEvents()
lick_events.create_timeseries(
'LickEvents',
data=np.ravel(matfile['lickx']),
timestamps=np.ravel(matfile['lickt']),
unit='centimeter',
description='Subject position in virtual hallway during the lick.')
nwbfile.add_acquisition(lick_events)
# Add information on the visual stimulus that was shown to the subject
# Assumed rate=60 [Hz]. Update if necessary
# Update external_file to link to Unity environment file
visualization = ImageSeries(
name='ImageSeries',
unit='seconds',
format='external',
external_file=list(['https://unity.com/VR-and-AR-corner']),
starting_time=vr_session_offset,
starting_frame=[[0]],
rate=float(60),
description='virtual Unity environment that the mouse navigates through'
)
nwbfile.add_stimulus(visualization)
# Add the recording device, a neuropixel probe
recording_device = nwbfile.create_device(name='neuropixel_probes')
electrode_group_description = 'single neuropixels probe http://www.open-ephys.org/neuropixelscorded'
electrode_group_name = 'probe1'
electrode_group = nwbfile.create_electrode_group(
electrode_group_name,
description=electrode_group_description,
location=subject_brain_region,
device=recording_device)
# Add information about each electrode
xcoords = np.ravel(matfile['sp'][0]['xcoords'][0])
ycoords = np.ravel(matfile['sp'][0]['ycoords'][0])
data_filtered_flag = matfile['sp'][0]['hp_filtered'][0][0]
if data_filtered_flag:
filter_desc = 'The raw voltage signals from the electrodes were high-pass filtered'
else:
filter_desc = 'The raw voltage signals from the electrodes were not high-pass filtered'
num_recording_electrodes = xcoords.shape[0]
recording_electrodes = range(0, num_recording_electrodes)
# create electrode columns for the x,y location on the neuropixel probe
# the standard x,y,z locations are reserved for Allen Brain Atlas location
nwbfile.add_electrode_column('rel_x', 'electrode x-location on the probe')
nwbfile.add_electrode_column('rel_y', 'electrode y-location on the probe')
for idx in recording_electrodes:
nwbfile.add_electrode(id=idx,
x=np.nan,
y=np.nan,
z=np.nan,
rel_x=float(xcoords[idx]),
rel_y=float(ycoords[idx]),
imp=np.nan,
location='medial entorhinal cortex',
filtering=filter_desc,
group=electrode_group)
# Add information about each unit, termed 'cluster' in giocomo data
# create new columns in unit table
nwbfile.add_unit_column(
'quality',
'labels given to clusters during manual sorting in phy (1=MUA, '
'2=Good, 3=Unsorted)')
# cluster information
cluster_ids = matfile['sp'][0]['cids'][0][0]
cluster_quality = matfile['sp'][0]['cgs'][0][0]
# spikes in time
spike_times = np.ravel(matfile['sp'][0]['st'][0]) # the time of each spike
spike_cluster = np.ravel(
matfile['sp'][0]['clu'][0]) # the cluster_id that spiked at that time
for i, cluster_id in enumerate(cluster_ids):
unit_spike_times = spike_times[spike_cluster == cluster_id]
waveforms = matfile['sp'][0]['temps'][0][cluster_id]
nwbfile.add_unit(id=int(cluster_id),
spike_times=unit_spike_times,
quality=cluster_quality[i],
waveform_mean=waveforms,
electrode_group=electrode_group)
# Trying to add another Units table to hold the results of the automatic spike sorting
# create TemplateUnits units table
template_units = Units(
name='TemplateUnits',
description='units assigned during automatic spike sorting')
template_units.add_column(
'tempScalingAmps',
'scaling amplitude applied to the template when extracting spike',
index=True)
# information on extracted spike templates
spike_templates = np.ravel(matfile['sp'][0]['spikeTemplates'][0])
spike_template_ids = np.unique(spike_templates)
# template scaling amplitudes
temp_scaling_amps = np.ravel(matfile['sp'][0]['tempScalingAmps'][0])
for i, spike_template_id in enumerate(spike_template_ids):
template_spike_times = spike_times[spike_templates ==
spike_template_id]
temp_scaling_amps_per_template = temp_scaling_amps[spike_templates ==
spike_template_id]
template_units.add_unit(id=int(spike_template_id),
spike_times=template_spike_times,
electrode_group=electrode_group,
tempScalingAmps=temp_scaling_amps_per_template)
# create ecephys processing module
spike_template_module = nwbfile.create_processing_module(
name='ecephys',
description='units assigned during automatic spike sorting')
# add template_units table to processing module
spike_template_module.add(template_units)
print(nwbfile)
print('converted to NWB:N')
print('saving ...')
with NWBHDF5IO(outpath, 'w') as io:
io.write(nwbfile)
print('saved', outpath)
# using external data. only the file paths will be stored inside the NWB file
image_series2 = TwoPhotonSeries(name='TwoPhotonSeries2',
dimension=[100, 100],
external_file=['images.tiff'],
imaging_plane=imaging_plane,
starting_frame=[0],
format='external',
starting_time=0.0,
rate=1.0)
####################
# Since these two-photon data are raw, acquired data, we will add the
# :py:class:`~pynwb.ophys.TwoPhotonSeries` objects to the :py:class:`~pynwb.NWBFile`
# as acquired data.
nwbfile.add_acquisition(image_series1)
nwbfile.add_acquisition(image_series2)
####################
# Motion Correction (optional)
# ---------------------------------
#
# You can also store the result of motion correction.
# These should be stored in a :py:class:`~pynwb.ophys.MotionCorrection` object,
# which is a :py:class:`~pynwb.core.MultiContainerInterface` (similar to pynwb.behavior.Position)
# which holds 1 or more :py:class:`~pynwb.ophys.CorrectedImageStack` objects.
corrected = ImageSeries(
name='corrected', # this must be named "corrected"
data=np.ones((1000, 100, 100)),
unit='na',
ephys_timestamps = np.arange(data_len) / rate
ephys_ts = ElectricalSeries(
'test_ephys_data',
'an hypothetical source',
ephys_data,
electrode_table_region,
timestamps=ephys_timestamps,
# Alternatively, could specify starting_time and rate as follows
# starting_time=ephys_timestamps[0],
# rate=rate,
resolution=0.001,
comments=
"This data was randomly generated with numpy, using 1234 as the seed",
description="Random numbers generated with numpy.random.rand")
nwbfile.add_acquisition(ephys_ts)
#######################
# Designating electrophysiology data
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
#
# As mentioned above, :py:class:`~pynwb.ecephys.ElectricalSeries` and :py:class:`~pynwb.ecephys.SpikeEventSeries`
# are meant for storing specific types of extracellular recordings. In addition to these two
# :py:class:`~pynwb.base.TimeSeries` classes, NWB provides some :ref:`data interfaces <basic_data_interfaces>`
# for designating the type of data you are storing. We will briefly discuss them here, and refer the reader to
# :py:mod:`API documentation <pynwb.ecephys>` and :ref:`PyNWB Basics tutorial <basics>` for more details on
# using these objects.
#
# For storing spike data, there are two options. Which one you choose depends on what data you have available.
# If you need to store the raw voltage traces, you should store your the traces with
# :py:class:`~pynwb.ecephys.ElectricalSeries` objects as :ref:`acquisition <basic_timeseries>` data, and use
def run_conversion(self, nwbfile: NWBFile, metadata: dict):
"""
Run conversion for this data interface.
Reads labview experiment behavioral data and adds it to nwbfile.
Parameters
----------
nwbfile : NWBFile
metadata : dict
"""
print("Converting Labview data...")
# Get list of trial summary files
dir_behavior_labview = self.source_data['dir_behavior_labview']
all_files = os.listdir(dir_behavior_labview)
trials_files = [f for f in all_files if '_sum.txt' in f]
trials_files.sort()
# Get session_start_time from first file timestamps
fpath = os.path.join(dir_behavior_labview, trials_files[0])
colnames = [
'Trial', 'StartT', 'EndT', 'Result', 'InitT', 'SpecificResults',
'ProbLeft', 'OptoDur', 'LRew', 'RRew', 'InterT', 'LTrial',
'ReactionTime', 'OptoCond', 'OptoTrial'
]
df_0 = pd.read_csv(fpath, sep='\t', index_col=False, names=colnames)
t0 = df_0['StartT'][0] # initial time in Labview seconds
# Add trials
print("Converting Labview trials data...")
if nwbfile.trials is not None:
print(
'Trials already exist in current nwb file. Labview behavior trials not added.'
)
else:
# Make dataframe
frames = []
for f in trials_files:
fpath = os.path.join(dir_behavior_labview, f)
frames.append(
pd.read_csv(fpath,
sep='\t',
index_col=False,
names=colnames))
df_trials_summary = pd.concat(frames)
nwbfile.add_trial_column(
name='results',
description=
"0 means sucess (rewarded trial), 1 means licks during intitial "
"period, which leads to a failed trial. 2 means early lick failure. 3 means "
"wrong lick or no response.")
nwbfile.add_trial_column(
name='init_t', description="duration of initial delay period.")
nwbfile.add_trial_column(
name='specific_results',
description=
"Possible outcomes classified based on raw data & meta file (_tr.m)."
)
nwbfile.add_trial_column(
name='prob_left',
description=
"probability for left trials in order to keep the number of "
"left and right trials balanced within the session. ")
nwbfile.add_trial_column(
name='opto_dur',
description="the duration of optical stimulation.")
nwbfile.add_trial_column(
name='l_rew_n',
description="counting the number of left rewards.")
nwbfile.add_trial_column(
name='r_rew_n',
description="counting the number of rightrewards.")
nwbfile.add_trial_column(name='inter_t',
description="inter-trial delay period.")
nwbfile.add_trial_column(
name='l_trial',
description=
"trial type (which side the air-puff is applied). 1 means "
"left-trial, 0 means right-trial")
nwbfile.add_trial_column(
name='reaction_time',
description=
"if it is a successful trial or wrong lick during response "
"period trial: ReactionTime = time between the first decision "
"lick and the beginning of the response period. If it is a failed "
"trial due to early licks: reaction time = the duration of "
"the air-puff period (in other words, when the animal licks "
"during the sample period).")
nwbfile.add_trial_column(
name='opto_cond',
description="0: no opto. 1: opto is on during sample period. "
"2: opto is on half way through the sample period (0.5s) "
"and 0.5 during the response period. 3. opto is on during "
"the response period.")
nwbfile.add_trial_column(
name='opto_trial',
description="1: opto trials. 0: Non-opto trials.")
for index, row in df_trials_summary.iterrows():
nwbfile.add_trial(
start_time=row['StartT'] - t0,
stop_time=row['EndT'] - t0,
results=int(row['Result']),
init_t=row['InitT'],
specific_results=int(row['SpecificResults']),
prob_left=row['ProbLeft'],
opto_dur=row['OptoDur'],
l_rew_n=int(row['LRew']),
r_rew_n=int(row['RRew']),
inter_t=row['InterT'],
l_trial=int(row['LTrial']),
reaction_time=int(row['ReactionTime']),
opto_cond=int(row['OptoCond']),
opto_trial=int(row['OptoTrial']),
)
# Get list of files: continuous data
continuous_files = [f.replace('_sum', '') for f in trials_files]
# Adds continuous behavioral data
frames = []
for f in continuous_files:
fpath_lick = os.path.join(dir_behavior_labview, f)
frames.append(pd.read_csv(fpath_lick, sep='\t', index_col=False))
df_continuous = pd.concat(frames)
# Behavioral data
print("Converting Labview behavior data...")
l1_ts = TimeSeries(name="left_lick",
data=df_continuous['Lick 1'].to_numpy(),
timestamps=df_continuous['Time'].to_numpy() - t0,
description="no description")
l2_ts = TimeSeries(name="right_lick",
data=df_continuous['Lick 2'].to_numpy(),
timestamps=df_continuous['Time'].to_numpy() - t0,
description="no description")
nwbfile.add_acquisition(l1_ts)
nwbfile.add_acquisition(l2_ts)
# Optogenetics stimulation data
print("Converting Labview optogenetics data...")
ogen_device = nwbfile.create_device(
name=metadata['Ogen']['Device']['name'],
description=metadata['Ogen']['Device']['description'])
meta_ogen_site = metadata['Ogen']['OptogeneticStimulusSite']
ogen_stim_site = OptogeneticStimulusSite(
name=meta_ogen_site['name'],
device=ogen_device,
description=meta_ogen_site['description'],
excitation_lambda=float(meta_ogen_site['excitation_lambda']),
location=meta_ogen_site['location'])
nwbfile.add_ogen_site(ogen_stim_site)
meta_ogen_series = metadata['Ogen']['OptogeneticSeries']
ogen_series = OptogeneticSeries(
name=meta_ogen_series['name'],
data=df_continuous['Opto'].to_numpy(),
site=ogen_stim_site,
timestamps=df_continuous['Time'].to_numpy() - t0,
description=meta_ogen_series['description'],
)
nwbfile.add_stimulus(ogen_series)
