def test_nwb_extractor(self):
path1 = self.test_dir + '/test.nwb'
se.NwbRecordingExtractor.write_recording(self.RX, path1)
RX_nwb = se.NwbRecordingExtractor(path1)
check_recording_return_types(RX_nwb)
check_recordings_equal(self.RX, RX_nwb)
check_dumping(RX_nwb)
del RX_nwb
# overwrite
se.NwbRecordingExtractor.write_recording(recording=self.RX,
save_path=path1)
RX_nwb = se.NwbRecordingExtractor(path1)
check_recording_return_types(RX_nwb)
check_recordings_equal(self.RX, RX_nwb)
check_dumping(RX_nwb)
# add sorting to existing
se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path1)
# create new
path2 = self.test_dir + '/firings_true.nwb'
se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path2)
SX_nwb = se.NwbSortingExtractor(path1)
check_sortings_equal(self.SX, SX_nwb)
check_dumping(SX_nwb)
def test_convert_recording_extractor_to_nwb(self, se_class, dataset_path, se_kwargs):
print(f"\n\n\n TESTING {se_class.extractor_name}...")
dataset_stem = Path(dataset_path).stem
self.dataset.get(dataset_path)
recording = se_class(**se_kwargs)
# # test writing to NWB
if test_nwb:
nwb_save_path = self.savedir / f"{se_class.__name__}_test_{dataset_stem}.nwb"
se.NwbRecordingExtractor.write_recording(recording, nwb_save_path, write_scaled=True)
nwb_recording = se.NwbRecordingExtractor(nwb_save_path)
check_recordings_equal(recording, nwb_recording)
if recording.has_unscaled:
nwb_save_path_unscaled = self.savedir / f"{se_class.__name__}_test_{dataset_stem}_unscaled.nwb"
if np.all(recording.get_channel_offsets() == 0):
se.NwbRecordingExtractor.write_recording(recording, nwb_save_path_unscaled, write_scaled=False)
nwb_recording = se.NwbRecordingExtractor(nwb_save_path_unscaled)
check_recordings_equal(recording, nwb_recording, return_scaled=False)
# Skip check when NWB converts uint to int
if recording.get_dtype(return_scaled=False) == nwb_recording.get_dtype(return_scaled=False):
check_recordings_equal(recording, nwb_recording, return_scaled=True)
# test caching
if test_caching:
rec_cache = se.CacheRecordingExtractor(recording)
check_recordings_equal(recording, rec_cache)
if recording.has_unscaled:
rec_cache_unscaled = se.CacheRecordingExtractor(recording, return_scaled=False)
check_recordings_equal(recording, rec_cache_unscaled, return_scaled=False)
check_recordings_equal(recording, rec_cache_unscaled, return_scaled=True)
def test_nwb_extractor(self):
path1 = self.test_dir + '/test.nwb'
se.NwbRecordingExtractor.write_recording(self.RX, path1)
RX_nwb = se.NwbRecordingExtractor(path1)
check_recording_return_types(RX_nwb)
check_recordings_equal(self.RX, RX_nwb)
check_dumping(RX_nwb)
del RX_nwb
se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1, overwrite=True)
RX_nwb = se.NwbRecordingExtractor(path1)
check_recording_return_types(RX_nwb)
check_recordings_equal(self.RX, RX_nwb)
check_dumping(RX_nwb)
# append sorting to existing file
se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path1, overwrite=False)
path2 = self.test_dir + "/firings_true.nwb"
se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path2)
se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path2)
SX_nwb = se.NwbSortingExtractor(path2)
check_sortings_equal(self.SX, SX_nwb)
check_dumping(SX_nwb)
# Test for handling unit property descriptions argument
property_descriptions = dict(stability="This is a description of stability.")
se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1, overwrite=True)
se.NwbSortingExtractor.write_sorting(
sorting=self.SX,
save_path=path1,
property_descriptions=property_descriptions
)
SX_nwb = se.NwbSortingExtractor(path1)
check_sortings_equal(self.SX, SX_nwb)
check_dumping(SX_nwb)
# Test for handling skip_properties argument
se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1, overwrite=True)
se.NwbSortingExtractor.write_sorting(
sorting=self.SX,
save_path=path1,
skip_properties=['stability']
)
SX_nwb = se.NwbSortingExtractor(path1)
assert 'stability' not in SX_nwb.get_shared_unit_property_names()
check_sortings_equal(self.SX, SX_nwb)
check_dumping(SX_nwb)
# Test for handling skip_features argument
se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1, overwrite=True)
se.NwbSortingExtractor.write_sorting(
sorting=self.SX2,
save_path=path1,
skip_features=['widths']
)
SX_nwb = se.NwbSortingExtractor(path1)
assert 'widths' not in SX_nwb.get_shared_unit_spike_feature_names()
check_sortings_equal(self.SX2, SX_nwb)
check_dumping(SX_nwb)
def test_nwb_extractor(self):
path1 = self.test_dir + '/test.nwb'
se.NwbRecordingExtractor.write_recording(self.RX, path1)
RX_nwb = se.NwbRecordingExtractor(path1)
check_recording_return_types(RX_nwb)
check_recordings_equal(self.RX, RX_nwb)
check_dumping(RX_nwb)
del RX_nwb
# overwrite
se.NwbRecordingExtractor.write_recording(recording=self.RX,
save_path=path1,
overwrite=True)
RX_nwb = se.NwbRecordingExtractor(path1)
check_recording_return_types(RX_nwb)
check_recordings_equal(self.RX, RX_nwb)
check_dumping(RX_nwb)
# add sorting to existing
se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path1)
# create new
path2 = self.test_dir + '/firings_true.nwb'
se.NwbRecordingExtractor.write_recording(recording=self.RX,
save_path=path2)
se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path2)
SX_nwb = se.NwbSortingExtractor(path2)
check_sortings_equal(self.SX, SX_nwb)
check_dumping(SX_nwb)
# Test for handling unit property descriptions argument
property_descriptions = {
'stability': 'this is a description of stability'
}
se.NwbSortingExtractor.write_sorting(
sorting=self.SX,
save_path=path1,
property_descriptions=property_descriptions)
# create new
se.NwbRecordingExtractor.write_recording(recording=self.RX,
save_path=path2,
overwrite=True)
se.NwbSortingExtractor.write_sorting(
sorting=self.SX,
save_path=path2,
property_descriptions=property_descriptions)
SX_nwb = se.NwbSortingExtractor(path2)
check_sortings_equal(self.SX, SX_nwb)
check_dumping(SX_nwb)
def test_write_recording(self):
path = self.test_dir + '/test.nwb'
write_recording(self.RX, path)
RX_nwb = se.NwbRecordingExtractor(path)
check_recording_return_types(RX_nwb)
check_recordings_equal(self.RX, RX_nwb)
check_dumping(RX_nwb)
del RX_nwb
write_recording(recording=self.RX, save_path=path, overwrite=True)
RX_nwb = se.NwbRecordingExtractor(path)
check_recording_return_types(RX_nwb)
check_recordings_equal(self.RX, RX_nwb)
check_dumping(RX_nwb)
# Writing multiple recordings using metadata
metadata = get_default_nwbfile_metadata()
path_multi = self.test_dir + '/test_multiple.nwb'
write_recording(
recording=self.RX,
save_path=path_multi,
metadata=metadata,
write_as='raw',
es_key='ElectricalSeries_raw',
)
write_recording(
recording=self.RX2,
save_path=path_multi,
metadata=metadata,
write_as='processed',
es_key='ElectricalSeries_processed',
)
write_recording(
recording=self.RX3,
save_path=path_multi,
metadata=metadata,
write_as='lfp',
es_key='ElectricalSeries_lfp',
)
RX_nwb = se.NwbRecordingExtractor(file_path=path_multi, electrical_series_name='raw_traces')
check_recording_return_types(RX_nwb)
check_recordings_equal(self.RX, RX_nwb)
check_dumping(RX_nwb)
del RX_nwb
def test_nwb_extractor(self):
path1 = self.test_dir + '/test.nwb'
se.NwbRecordingExtractor.write_recording(self.RX, path1)
RX_nwb = se.NwbRecordingExtractor(path1)
self._check_recording_return_types(RX_nwb)
self._check_recordings_equal(self.RX, RX_nwb)
del RX_nwb
# overwrite
se.NwbRecordingExtractor.write_recording(self.RX,
path1,
session_description='second',
identifier='19475')
RX_nwb = se.NwbRecordingExtractor(path1)
self._check_recording_return_types(RX_nwb)
self._check_recordings_equal(self.RX, RX_nwb)
# add sorting to existing
se.NwbSortingExtractor.write_sorting(self.SX, path1)
# create new
path2 = self.test_dir + '/firings_true.nwb'
se.NwbSortingExtractor.write_sorting(self.SX,
path2,
session_description='second',
identifier='19475')
def javascript_state_changed(self, prev_state, state):
self.set_python_state(dict(status='running', status_message='Running'))
mt.configDownloadFrom(state.get('download_from', []))
path = state.get('path', None)
if path:
self.set_python_state(dict(status_message='Realizing file: {}'.format(path)))
if path.endswith('.csv'):
path2 = mt.realizeFile(path)
if not path2:
self.set_python_state(dict(
status='error',
status_message='Unable to realize file: {}'.format(path)
))
return
self.set_python_state(dict(status_message='Loading locatoins'))
x = np.genfromtxt(path2, delimiter=',')
locations = x.T
num_elec = x.shape[0]
labels = ['{}'.format(a) for a in range(1, num_elec + 1)]
elif path.endswith('.nwb'):
path2 = mt.realizeFile(path)
if not path2:
self.set_python_state(dict(
status='error',
status_message='Unable to realize file: {}'.format(path)
))
return
X = se.NwbRecordingExtractor(path=path2)
locations = X.get_channel_locations()
num_elec = locations.shape[0]
labels = ['{}'.format(a) for a in range(1, num_elec + 1)]
else:
locations = [[0, 0], [1, 0], [1, 1], [2, 1]]
labels = ['1', '2', '3', '4']
state = dict()
state['locations'] = locations
state['labels'] = labels
state['status'] = 'finished'
self.set_python_state(state)
def get_recording_extractor(self, key, sort_interval):
"""Given a key containing the key fields for a SpikeSorting schema, and the interval to be sorted,
returns the recording extractor object (see the spikeinterface package for details)
:param key: key to SpikeSorting schema
:type key: dict
:param sort_interval: [start_time, end_time]
:type sort_interval: 1D array with the start and end times for this sort
:return: (recording_extractor, sort_interval_valid_times)
:rtype: tuple with spikeextractor recording extractor object and valid times list
"""
interval_list_name = (SpikeSortingParameters() & key).fetch1('interval_list_name')
valid_times = (IntervalList() & {'nwb_file_name' : key['nwb_file_name'],
'interval_list_name' : interval_list_name})\
.fetch('valid_times')[0]
sort_interval_valid_times = interval_list_intersect(np.array([sort_interval]), valid_times)
raw_data_obj = (Raw() & {'nwb_file_name' : key['nwb_file_name']}).fetch_nwb()[0]['raw']
# get the indeces of the data to use. Note that spike_extractors has a time_to_frame function,
# but it seems to set the time of the first sample to 0, which will not match our intervals
timestamps = np.asarray(raw_data_obj.timestamps)
sort_indeces = np.searchsorted(timestamps, np.ravel(sort_interval))
#print(f'sample indeces: {sort_indeces}')
# Use spike_interface to run the sorter on the selected sort group
raw_data = se.NwbRecordingExtractor(Nwbfile.get_abs_path(key['nwb_file_name']), electrical_series_name='e-series')
# Blank out non-valid times.
exclude_inds = interval_list_excludes_ind(sort_interval_valid_times, timestamps)
exclude_inds = exclude_inds[exclude_inds <= sort_indeces[-1]]
# TODO: add a blanking function to the preprocessing module
raw_data = st.preprocessing.remove_artifacts(raw_data, exclude_inds, ms_before=0.1, ms_after=0.1)
# create a group id within spikeinterface for the specified electodes
electrode_ids = (SortGroup.SortGroupElectrode() & {'nwb_file_name' : key['nwb_file_name'],
'sort_group_id' : key['sort_group_id']}).fetch('electrode_id')
raw_data.set_channel_groups([key['sort_group_id']]*len(electrode_ids), channel_ids=electrode_ids)
raw_data.add_epoch(key['sort_interval_list_name'], sort_indeces[0], sort_indeces[1])
# restrict the raw data to the specific samples
raw_data_epoch = raw_data.get_epoch(key['sort_interval_list_name'])
# get the reference for this sort group
sort_reference_electrode_id = (SortGroup() & {'nwb_file_name' : key['nwb_file_name'],
'sort_group_id' : key['sort_group_id']}
).fetch('sort_reference_electrode_id')
if sort_reference_electrode_id >= 0:
raw_data_epoch_referenced = st.preprocessing.common_reference(raw_data_epoch, reference='single',
groups=[key['sort_group_id']], ref_channels=sort_reference_electrode_id)
elif sort_reference_electrode_id == -2:
raw_data_epoch_referenced = st.preprocessing.common_reference(raw_data, reference='median')
else:
raw_data_epoch_referenced = raw_data_epoch
# create a temporary file for the probe with a .prb extension and write out the channel locations in the prb file
with tempfile.TemporaryDirectory() as tmp_dir:
prb_file_name = os.path.join(tmp_dir, 'sortgroup.prb')
SortGroup().write_prb(key['sort_group_id'], key['nwb_file_name'], prb_file_name)
# add the probe geometry to the raw_data recording
raw_data_epoch_referenced.load_probe_file(prb_file_name)
return se.SubRecordingExtractor(raw_data_epoch_referenced,channel_ids=electrode_ids), sort_interval_valid_times
def test_nwb_extractor(self):
path1 = self.test_dir + '/test.nwb'
se.NwbRecordingExtractor.write_recording(self.RX, path1)
RX_nwb = se.NwbRecordingExtractor(path1)
check_recording_return_types(RX_nwb)
check_recordings_equal(self.RX, RX_nwb)
check_dumping(RX_nwb)
del RX_nwb
se.NwbRecordingExtractor.write_recording(recording=self.RX,
save_path=path1,
overwrite=True)
RX_nwb = se.NwbRecordingExtractor(path1)
check_recording_return_types(RX_nwb)
check_recordings_equal(self.RX, RX_nwb)
check_dumping(RX_nwb)
# append sorting to existing file
se.NwbSortingExtractor.write_sorting(sorting=self.SX,
save_path=path1,
overwrite=False)
path2 = self.test_dir + "/firings_true.nwb"
se.NwbRecordingExtractor.write_recording(recording=self.RX,
save_path=path2)
se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path2)
SX_nwb = se.NwbSortingExtractor(path2)
check_sortings_equal(self.SX, SX_nwb)
check_dumping(SX_nwb)
# Test for handling unit property descriptions argument
property_descriptions = dict(
stability="This is a description of stability.")
se.NwbRecordingExtractor.write_recording(recording=self.RX,
save_path=path1,
overwrite=True)
se.NwbSortingExtractor.write_sorting(
sorting=self.SX,
save_path=path1,
property_descriptions=property_descriptions)
SX_nwb = se.NwbSortingExtractor(path1)
check_sortings_equal(self.SX, SX_nwb)
check_dumping(SX_nwb)
# Test for handling skip_properties argument
se.NwbRecordingExtractor.write_recording(recording=self.RX,
save_path=path1,
overwrite=True)
se.NwbSortingExtractor.write_sorting(sorting=self.SX,
save_path=path1,
skip_properties=['stability'])
SX_nwb = se.NwbSortingExtractor(path1)
assert 'stability' not in SX_nwb.get_shared_unit_property_names()
check_sortings_equal(self.SX, SX_nwb)
check_dumping(SX_nwb)
# Test for handling skip_features argument
se.NwbRecordingExtractor.write_recording(recording=self.RX,
save_path=path1,
overwrite=True)
# SX2 has timestamps, so loading it back from Nwb will not recover the same spike frames. USe use_times=False
se.NwbSortingExtractor.write_sorting(sorting=self.SX2,
save_path=path1,
skip_features=['widths'],
use_times=False)
SX_nwb = se.NwbSortingExtractor(path1)
assert 'widths' not in SX_nwb.get_shared_unit_spike_feature_names()
check_sortings_equal(self.SX2, SX_nwb)
check_dumping(SX_nwb)
# Test writting multiple recordings using metadata
metadata = get_default_nwbfile_metadata()
path_nwb = self.test_dir + '/test_multiple.nwb'
se.NwbRecordingExtractor.write_recording(
recording=self.RX,
save_path=path_nwb,
metadata=metadata,
write_as='raw',
es_key='ElectricalSeries_raw',
)
se.NwbRecordingExtractor.write_recording(
recording=self.RX2,
save_path=path_nwb,
metadata=metadata,
write_as='processed',
es_key='ElectricalSeries_processed',
)
se.NwbRecordingExtractor.write_recording(
recording=self.RX3,
save_path=path_nwb,
metadata=metadata,
write_as='lfp',
es_key='ElectricalSeries_lfp',
)
RX_nwb = se.NwbRecordingExtractor(file_path=path_nwb,
electrical_series_name='raw_traces')
check_recording_return_types(RX_nwb)
check_recordings_equal(self.RX, RX_nwb)
check_dumping(RX_nwb)
del RX_nwb
