def test_nwb_extractor(self): path1 = self.test_dir + '/test.nwb' se.NwbRecordingExtractor.write_recording(self.RX, path1) RX_nwb = se.NwbRecordingExtractor(path1) check_recording_return_types(RX_nwb) check_recordings_equal(self.RX, RX_nwb) check_dumping(RX_nwb) del RX_nwb # overwrite se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1) RX_nwb = se.NwbRecordingExtractor(path1) check_recording_return_types(RX_nwb) check_recordings_equal(self.RX, RX_nwb) check_dumping(RX_nwb) # add sorting to existing se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path1) # create new path2 = self.test_dir + '/firings_true.nwb' se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path2) SX_nwb = se.NwbSortingExtractor(path1) check_sortings_equal(self.SX, SX_nwb) check_dumping(SX_nwb)
def test_convert_recording_extractor_to_nwb(self, se_class, dataset_path, se_kwargs): print(f"\n\n\n TESTING {se_class.extractor_name}...") dataset_stem = Path(dataset_path).stem self.dataset.get(dataset_path) recording = se_class(**se_kwargs) # # test writing to NWB if test_nwb: nwb_save_path = self.savedir / f"{se_class.__name__}_test_{dataset_stem}.nwb" se.NwbRecordingExtractor.write_recording(recording, nwb_save_path, write_scaled=True) nwb_recording = se.NwbRecordingExtractor(nwb_save_path) check_recordings_equal(recording, nwb_recording) if recording.has_unscaled: nwb_save_path_unscaled = self.savedir / f"{se_class.__name__}_test_{dataset_stem}_unscaled.nwb" if np.all(recording.get_channel_offsets() == 0): se.NwbRecordingExtractor.write_recording(recording, nwb_save_path_unscaled, write_scaled=False) nwb_recording = se.NwbRecordingExtractor(nwb_save_path_unscaled) check_recordings_equal(recording, nwb_recording, return_scaled=False) # Skip check when NWB converts uint to int if recording.get_dtype(return_scaled=False) == nwb_recording.get_dtype(return_scaled=False): check_recordings_equal(recording, nwb_recording, return_scaled=True) # test caching if test_caching: rec_cache = se.CacheRecordingExtractor(recording) check_recordings_equal(recording, rec_cache) if recording.has_unscaled: rec_cache_unscaled = se.CacheRecordingExtractor(recording, return_scaled=False) check_recordings_equal(recording, rec_cache_unscaled, return_scaled=False) check_recordings_equal(recording, rec_cache_unscaled, return_scaled=True)
def test_nwb_extractor(self): path1 = self.test_dir + '/test.nwb' se.NwbRecordingExtractor.write_recording(self.RX, path1) RX_nwb = se.NwbRecordingExtractor(path1) check_recording_return_types(RX_nwb) check_recordings_equal(self.RX, RX_nwb) check_dumping(RX_nwb) del RX_nwb se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1, overwrite=True) RX_nwb = se.NwbRecordingExtractor(path1) check_recording_return_types(RX_nwb) check_recordings_equal(self.RX, RX_nwb) check_dumping(RX_nwb) # append sorting to existing file se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path1, overwrite=False) path2 = self.test_dir + "/firings_true.nwb" se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path2) se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path2) SX_nwb = se.NwbSortingExtractor(path2) check_sortings_equal(self.SX, SX_nwb) check_dumping(SX_nwb) # Test for handling unit property descriptions argument property_descriptions = dict(stability="This is a description of stability.") se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1, overwrite=True) se.NwbSortingExtractor.write_sorting( sorting=self.SX, save_path=path1, property_descriptions=property_descriptions ) SX_nwb = se.NwbSortingExtractor(path1) check_sortings_equal(self.SX, SX_nwb) check_dumping(SX_nwb) # Test for handling skip_properties argument se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1, overwrite=True) se.NwbSortingExtractor.write_sorting( sorting=self.SX, save_path=path1, skip_properties=['stability'] ) SX_nwb = se.NwbSortingExtractor(path1) assert 'stability' not in SX_nwb.get_shared_unit_property_names() check_sortings_equal(self.SX, SX_nwb) check_dumping(SX_nwb) # Test for handling skip_features argument se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1, overwrite=True) se.NwbSortingExtractor.write_sorting( sorting=self.SX2, save_path=path1, skip_features=['widths'] ) SX_nwb = se.NwbSortingExtractor(path1) assert 'widths' not in SX_nwb.get_shared_unit_spike_feature_names() check_sortings_equal(self.SX2, SX_nwb) check_dumping(SX_nwb)
def test_nwb_extractor(self): path1 = self.test_dir + '/test.nwb' se.NwbRecordingExtractor.write_recording(self.RX, path1) RX_nwb = se.NwbRecordingExtractor(path1) check_recording_return_types(RX_nwb) check_recordings_equal(self.RX, RX_nwb) check_dumping(RX_nwb) del RX_nwb # overwrite se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1, overwrite=True) RX_nwb = se.NwbRecordingExtractor(path1) check_recording_return_types(RX_nwb) check_recordings_equal(self.RX, RX_nwb) check_dumping(RX_nwb) # add sorting to existing se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path1) # create new path2 = self.test_dir + '/firings_true.nwb' se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path2) se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path2) SX_nwb = se.NwbSortingExtractor(path2) check_sortings_equal(self.SX, SX_nwb) check_dumping(SX_nwb) # Test for handling unit property descriptions argument property_descriptions = { 'stability': 'this is a description of stability' } se.NwbSortingExtractor.write_sorting( sorting=self.SX, save_path=path1, property_descriptions=property_descriptions) # create new se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path2, overwrite=True) se.NwbSortingExtractor.write_sorting( sorting=self.SX, save_path=path2, property_descriptions=property_descriptions) SX_nwb = se.NwbSortingExtractor(path2) check_sortings_equal(self.SX, SX_nwb) check_dumping(SX_nwb)
def test_write_recording(self): path = self.test_dir + '/test.nwb' write_recording(self.RX, path) RX_nwb = se.NwbRecordingExtractor(path) check_recording_return_types(RX_nwb) check_recordings_equal(self.RX, RX_nwb) check_dumping(RX_nwb) del RX_nwb write_recording(recording=self.RX, save_path=path, overwrite=True) RX_nwb = se.NwbRecordingExtractor(path) check_recording_return_types(RX_nwb) check_recordings_equal(self.RX, RX_nwb) check_dumping(RX_nwb) # Writing multiple recordings using metadata metadata = get_default_nwbfile_metadata() path_multi = self.test_dir + '/test_multiple.nwb' write_recording( recording=self.RX, save_path=path_multi, metadata=metadata, write_as='raw', es_key='ElectricalSeries_raw', ) write_recording( recording=self.RX2, save_path=path_multi, metadata=metadata, write_as='processed', es_key='ElectricalSeries_processed', ) write_recording( recording=self.RX3, save_path=path_multi, metadata=metadata, write_as='lfp', es_key='ElectricalSeries_lfp', ) RX_nwb = se.NwbRecordingExtractor(file_path=path_multi, electrical_series_name='raw_traces') check_recording_return_types(RX_nwb) check_recordings_equal(self.RX, RX_nwb) check_dumping(RX_nwb) del RX_nwb
def test_nwb_extractor(self): path1 = self.test_dir + '/test.nwb' se.NwbRecordingExtractor.write_recording(self.RX, path1) RX_nwb = se.NwbRecordingExtractor(path1) self._check_recording_return_types(RX_nwb) self._check_recordings_equal(self.RX, RX_nwb) del RX_nwb # overwrite se.NwbRecordingExtractor.write_recording(self.RX, path1, session_description='second', identifier='19475') RX_nwb = se.NwbRecordingExtractor(path1) self._check_recording_return_types(RX_nwb) self._check_recordings_equal(self.RX, RX_nwb) # add sorting to existing se.NwbSortingExtractor.write_sorting(self.SX, path1) # create new path2 = self.test_dir + '/firings_true.nwb' se.NwbSortingExtractor.write_sorting(self.SX, path2, session_description='second', identifier='19475')
def javascript_state_changed(self, prev_state, state): self.set_python_state(dict(status='running', status_message='Running')) mt.configDownloadFrom(state.get('download_from', [])) path = state.get('path', None) if path: self.set_python_state(dict(status_message='Realizing file: {}'.format(path))) if path.endswith('.csv'): path2 = mt.realizeFile(path) if not path2: self.set_python_state(dict( status='error', status_message='Unable to realize file: {}'.format(path) )) return self.set_python_state(dict(status_message='Loading locatoins')) x = np.genfromtxt(path2, delimiter=',') locations = x.T num_elec = x.shape[0] labels = ['{}'.format(a) for a in range(1, num_elec + 1)] elif path.endswith('.nwb'): path2 = mt.realizeFile(path) if not path2: self.set_python_state(dict( status='error', status_message='Unable to realize file: {}'.format(path) )) return X = se.NwbRecordingExtractor(path=path2) locations = X.get_channel_locations() num_elec = locations.shape[0] labels = ['{}'.format(a) for a in range(1, num_elec + 1)] else: locations = [[0, 0], [1, 0], [1, 1], [2, 1]] labels = ['1', '2', '3', '4'] state = dict() state['locations'] = locations state['labels'] = labels state['status'] = 'finished' self.set_python_state(state)
def get_recording_extractor(self, key, sort_interval): """Given a key containing the key fields for a SpikeSorting schema, and the interval to be sorted, returns the recording extractor object (see the spikeinterface package for details) :param key: key to SpikeSorting schema :type key: dict :param sort_interval: [start_time, end_time] :type sort_interval: 1D array with the start and end times for this sort :return: (recording_extractor, sort_interval_valid_times) :rtype: tuple with spikeextractor recording extractor object and valid times list """ interval_list_name = (SpikeSortingParameters() & key).fetch1('interval_list_name') valid_times = (IntervalList() & {'nwb_file_name' : key['nwb_file_name'], 'interval_list_name' : interval_list_name})\ .fetch('valid_times')[0] sort_interval_valid_times = interval_list_intersect(np.array([sort_interval]), valid_times) raw_data_obj = (Raw() & {'nwb_file_name' : key['nwb_file_name']}).fetch_nwb()[0]['raw'] # get the indeces of the data to use. Note that spike_extractors has a time_to_frame function, # but it seems to set the time of the first sample to 0, which will not match our intervals timestamps = np.asarray(raw_data_obj.timestamps) sort_indeces = np.searchsorted(timestamps, np.ravel(sort_interval)) #print(f'sample indeces: {sort_indeces}') # Use spike_interface to run the sorter on the selected sort group raw_data = se.NwbRecordingExtractor(Nwbfile.get_abs_path(key['nwb_file_name']), electrical_series_name='e-series') # Blank out non-valid times. exclude_inds = interval_list_excludes_ind(sort_interval_valid_times, timestamps) exclude_inds = exclude_inds[exclude_inds <= sort_indeces[-1]] # TODO: add a blanking function to the preprocessing module raw_data = st.preprocessing.remove_artifacts(raw_data, exclude_inds, ms_before=0.1, ms_after=0.1) # create a group id within spikeinterface for the specified electodes electrode_ids = (SortGroup.SortGroupElectrode() & {'nwb_file_name' : key['nwb_file_name'], 'sort_group_id' : key['sort_group_id']}).fetch('electrode_id') raw_data.set_channel_groups([key['sort_group_id']]*len(electrode_ids), channel_ids=electrode_ids) raw_data.add_epoch(key['sort_interval_list_name'], sort_indeces[0], sort_indeces[1]) # restrict the raw data to the specific samples raw_data_epoch = raw_data.get_epoch(key['sort_interval_list_name']) # get the reference for this sort group sort_reference_electrode_id = (SortGroup() & {'nwb_file_name' : key['nwb_file_name'], 'sort_group_id' : key['sort_group_id']} ).fetch('sort_reference_electrode_id') if sort_reference_electrode_id >= 0: raw_data_epoch_referenced = st.preprocessing.common_reference(raw_data_epoch, reference='single', groups=[key['sort_group_id']], ref_channels=sort_reference_electrode_id) elif sort_reference_electrode_id == -2: raw_data_epoch_referenced = st.preprocessing.common_reference(raw_data, reference='median') else: raw_data_epoch_referenced = raw_data_epoch # create a temporary file for the probe with a .prb extension and write out the channel locations in the prb file with tempfile.TemporaryDirectory() as tmp_dir: prb_file_name = os.path.join(tmp_dir, 'sortgroup.prb') SortGroup().write_prb(key['sort_group_id'], key['nwb_file_name'], prb_file_name) # add the probe geometry to the raw_data recording raw_data_epoch_referenced.load_probe_file(prb_file_name) return se.SubRecordingExtractor(raw_data_epoch_referenced,channel_ids=electrode_ids), sort_interval_valid_times
def test_nwb_extractor(self): path1 = self.test_dir + '/test.nwb' se.NwbRecordingExtractor.write_recording(self.RX, path1) RX_nwb = se.NwbRecordingExtractor(path1) check_recording_return_types(RX_nwb) check_recordings_equal(self.RX, RX_nwb) check_dumping(RX_nwb) del RX_nwb se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1, overwrite=True) RX_nwb = se.NwbRecordingExtractor(path1) check_recording_return_types(RX_nwb) check_recordings_equal(self.RX, RX_nwb) check_dumping(RX_nwb) # append sorting to existing file se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path1, overwrite=False) path2 = self.test_dir + "/firings_true.nwb" se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path2) se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path2) SX_nwb = se.NwbSortingExtractor(path2) check_sortings_equal(self.SX, SX_nwb) check_dumping(SX_nwb) # Test for handling unit property descriptions argument property_descriptions = dict( stability="This is a description of stability.") se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1, overwrite=True) se.NwbSortingExtractor.write_sorting( sorting=self.SX, save_path=path1, property_descriptions=property_descriptions) SX_nwb = se.NwbSortingExtractor(path1) check_sortings_equal(self.SX, SX_nwb) check_dumping(SX_nwb) # Test for handling skip_properties argument se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1, overwrite=True) se.NwbSortingExtractor.write_sorting(sorting=self.SX, save_path=path1, skip_properties=['stability']) SX_nwb = se.NwbSortingExtractor(path1) assert 'stability' not in SX_nwb.get_shared_unit_property_names() check_sortings_equal(self.SX, SX_nwb) check_dumping(SX_nwb) # Test for handling skip_features argument se.NwbRecordingExtractor.write_recording(recording=self.RX, save_path=path1, overwrite=True) # SX2 has timestamps, so loading it back from Nwb will not recover the same spike frames. USe use_times=False se.NwbSortingExtractor.write_sorting(sorting=self.SX2, save_path=path1, skip_features=['widths'], use_times=False) SX_nwb = se.NwbSortingExtractor(path1) assert 'widths' not in SX_nwb.get_shared_unit_spike_feature_names() check_sortings_equal(self.SX2, SX_nwb) check_dumping(SX_nwb) # Test writting multiple recordings using metadata metadata = get_default_nwbfile_metadata() path_nwb = self.test_dir + '/test_multiple.nwb' se.NwbRecordingExtractor.write_recording( recording=self.RX, save_path=path_nwb, metadata=metadata, write_as='raw', es_key='ElectricalSeries_raw', ) se.NwbRecordingExtractor.write_recording( recording=self.RX2, save_path=path_nwb, metadata=metadata, write_as='processed', es_key='ElectricalSeries_processed', ) se.NwbRecordingExtractor.write_recording( recording=self.RX3, save_path=path_nwb, metadata=metadata, write_as='lfp', es_key='ElectricalSeries_lfp', ) RX_nwb = se.NwbRecordingExtractor(file_path=path_nwb, electrical_series_name='raw_traces') check_recording_return_types(RX_nwb) check_recordings_equal(self.RX, RX_nwb) check_dumping(RX_nwb) del RX_nwb