def __init__(self, path_to_recordings, path_to_geom=None, spike_size=None, neighbor_radius=None, dtype=None, n_channels=None, data_format=None, mmap=True, waveform_dtype='float32'): self.data = RecordingsReader(path_to_recordings, dtype, n_channels, data_format, mmap, output_shape='long') if path_to_geom is not None: self.geom = geom.parse(path_to_geom, n_channels) self.neighbor_radius = neighbor_radius self.neigh_matrix = geom.find_channel_neighbors( self.geom, neighbor_radius) self.n_channels = self.data.channels self.spike_size = spike_size if waveform_dtype == 'default': waveform_dtype = dtype self.waveform_dtype = waveform_dtype self.logger = logging.getLogger(__name__)
def __init__(self, path_to_recordings, path_to_geom=None, spike_size=None, neighbor_radius=None, dtype=None, n_channels=None, data_order=None, loader='memmap', waveform_dtype='float32'): self.reader = RecordingsReader(path_to_recordings, dtype, n_channels, data_order, loader) if path_to_geom is not None: self.geom = geom.parse(path_to_geom, n_channels) self.neighbor_radius = neighbor_radius self.neigh_matrix = geom.find_channel_neighbors( self.geom, neighbor_radius) self.n_channels = self.reader.channels self.spike_size = spike_size if waveform_dtype == 'default': waveform_dtype = dtype self.waveform_dtype = waveform_dtype self.logger = logging.getLogger(__name__)
def __init__(self, mapping): mapping = validate(mapping) super(Config, self).__init__(mapping) self._logger = logging.getLogger(__name__) # init the rest of the parameters, these parameters are used # througout the pipeline so we compute them once to avoid redudant # computations # GEOMETRY PARAMETERS path_to_geom = path.join(self.data.root_folder, self.data.geometry) self._set_param('geom', geom.parse(path_to_geom, self.recordings.n_channels)) neigh_channels = geom.find_channel_neighbors( self.geom, self.recordings.spatial_radius) self._set_param('neigh_channels', neigh_channels) channel_groups = geom.make_channel_groups(self.recordings.n_channels, self.neigh_channels, self.geom) self._set_param('channel_groups', channel_groups) self._set_param( 'spike_size', int( np.round(self.recordings.spike_size_ms * self.recordings.sampling_rate / (2 * 1000))))
def __init__(self, mapping, output_directory=None): self._logger = logging.getLogger(__name__) # FIXME: not raising errors due to schema validation for now mapping = validate(mapping, silent=True) _processes = mapping['resources']['processes'] mapping['resources']['processes'] = ( multiprocess.cpu_count() if _processes == 'max' else _processes) self._frozenjson = FrozenJSON(mapping) if output_directory is not None: if path.isabs(output_directory): self._path_to_output_directory = output_directory else: _ = Path(self.data.root_folder, output_directory) self._path_to_output_directory = str(_) else: self._path_to_output_directory = None # init the rest of the parameters, these parameters are used # througout the pipeline so we compute them once to avoid redudant # computations # GEOMETRY PARAMETERS path_to_geom = path.join(self.data.root_folder, self.data.geometry) self._set_param('geom', geom.parse(path_to_geom, self.recordings.n_channels)) # check dimensions of the geometry file n_channels_geom, _ = self.geom.shape if self.recordings.n_channels != n_channels_geom: raise ValueError('Channels in the geometry file ({}) does not ' 'value in the configuration file ({})'.format( n_channels_geom, self.recordings.n_channels)) neigh_channels = geom.find_channel_neighbors( self.geom, self.recordings.spatial_radius) self._set_param('neigh_channels', neigh_channels) channel_groups = geom.make_channel_groups(self.recordings.n_channels, self.neigh_channels, self.geom) self._set_param('channel_groups', channel_groups) self._set_param( 'spike_size', int( np.round(self.recordings.spike_size_ms * self.recordings.sampling_rate / (2 * 1000)))) channel_index = geom.make_channel_index(self.neigh_channels, self.geom, steps=2) self._set_param('channel_index', channel_index)
def test_can_use_threshold_detector(data, data_info, path_to_geometry): geometry = parse(path_to_geometry, data_info['recordings']['n_channels']) neighbors = find_channel_neighbors(geometry, radius=70) # FIXME: using the same formula from yass/config/config.py, might be # better to avoid having this hardcoded spike_size = int( np.round(data_info['recordings']['spike_size_ms'] * data_info['recordings']['sampling_rate'] / (2 * 1000))) detect._threshold(data, neighbors, spike_size, 5)
def test_can_compute_whiten_matrix(data, data_info, path_to_geometry): geometry = parse(path_to_geometry, data_info['recordings']['n_channels']) neighbors = find_channel_neighbors(geometry, radius=70) channel_index = make_channel_index(neighbors, geometry) # FIXME: using the same formula from yass/config/config.py, might be # better to avoid having this hardcoded spike_size = int( np.round(data_info['recordings']['spike_size_ms'] * data_info['recordings']['sampling_rate'] / (2 * 1000))) whiten._matrix(data, channel_index, spike_size)
def __init__(self, rec_file, geom_file, sample_rate, n_batches, batch_time_samples, n_chan, radius, scale=1e2, filter_std=True, whiten=True): """Sets up the object for reading from a binary file. Parameters ---------- rec_file: str Path to binary file that contains the raw recording file. geom_file: str Path to text file containing the geometry file. The file should contain n_chan lines and each line should contain two numbers that are separated by ' '. sample_rate: int Recording sample rate in Hz n_batches: int Processes the recording in n_batches number of consecuitive segments that start from the beginning. batch_time_samples: int Number of time samples per each batch to be used. filter_std: bool The iterator both filters and standardizes the recording (dividing by standard deviation. whiten: bool Spatially whiten the recording. scale: float In case filter and whitening is not needed and the binary data is scaled up. """ self.s_rate = sample_rate self.batch_time_samples = batch_time_samples self.n_batches = n_batches self.n_chan = n_chan self.radius = radius self.geometry = parse(geom_file, n_chan) self.neighbs = find_channel_neighbors(self.geometry, self.radius) self.filter_std = filter_std self.whiten = whiten self.scale = scale self.file = open(rec_file, 'r')
def __init__(self, mapping): mapping = validate(mapping) super(Config, self).__init__(mapping) self._logger = logging.getLogger(__name__) # init the rest of the parameters, these parameters are used # througout the pipeline so we compute them once to avoid redudant # computations # GEOMETRY PARAMETERS path_to_geom = path.join(self.data.root_folder, self.data.geometry) self._set_param('geom', geom.parse(path_to_geom, self.recordings.n_channels)) # check dimensions of the geometry file n_channels_geom, _ = self.geom.shape if self.recordings.n_channels != n_channels_geom: raise ValueError('Channels in the geometry file ({}) does not ' 'value in the configuration file ({})' .format(n_channels_geom, self.recordings.n_channels)) neigh_channels = geom.find_channel_neighbors( self.geom, self.recordings.spatial_radius) self._set_param('neigh_channels', neigh_channels) channel_groups = geom.make_channel_groups( self.recordings.n_channels, self.neigh_channels, self.geom) self._set_param('channel_groups', channel_groups) self._set_param( 'spike_size', int( np.round(self.recordings.spike_size_ms * self.recordings.sampling_rate / (2 * 1000)))) channel_index = geom.make_channel_index(self.neigh_channels, self.geom, steps=2) self._set_param('channel_index', channel_index)
def __init__(self, mapping, output_directory=None): self._logger = logging.getLogger(__name__) # FIXME: not raising errors due to schema validation for now mapping = validate(mapping, silent=True) self._frozenjson = FrozenJSON(mapping) if output_directory is not None: if path.isabs(output_directory): self._path_to_output_directory = output_directory else: _ = Path(self.data.root_folder, output_directory) self._path_to_output_directory = str(_) else: self._path_to_output_directory = None # init the rest of the parameters, these parameters are used # througout the pipeline so we compute them once to avoid redudant # computations # GEOMETRY PARAMETERS path_to_geom = path.join(self.data.root_folder, self.data.geometry) self._set_param('geom', geom.parse(path_to_geom, self.recordings.n_channels)) # check dimensions of the geometry file n_channels_geom, _ = self.geom.shape if self.recordings.n_channels != n_channels_geom: raise ValueError('Channels in the geometry file ({}) does not ' 'value in the configuration file ({})'.format( n_channels_geom, self.recordings.n_channels)) neigh_channels = geom.find_channel_neighbors( self.geom, self.recordings.spatial_radius) self._set_param('neigh_channels', neigh_channels) # spike size long (to cover full axonal propagation) spike_size = int( np.round(self.recordings.spike_size_ms * self.recordings.sampling_rate / 1000)) if spike_size % 2 == 0: spike_size += 1 self._set_param('spike_size', spike_size) # spike size center if self.recordings.center_spike_size_ms is not None: center_spike_size = int( np.round(self.recordings.center_spike_size_ms * self.recordings.sampling_rate / 1000)) if center_spike_size % 2 == 0: center_spike_size += 1 else: center_spike_size = int(np.copy(spike_size)) self._set_param('center_spike_size', center_spike_size) # channel index for nn channel_index = geom.make_channel_index(self.neigh_channels, self.geom, steps=1) self._set_param('channel_index', channel_index) # spike size to nn if self.neuralnetwork.apply_nn: if self.neuralnetwork.training.spike_size_ms is None: detect_saved_file = torch.load( self.neuralnetwork.detect.filename, map_location=lambda storage, loc: storage) spike_size_nn_detector = detect_saved_file[ 'temporal_filter1.0.weight'].shape[2] denoised_saved_file = torch.load( self.neuralnetwork.denoise.filename, map_location=lambda storage, loc: storage) spike_size_nn_denoiser = denoised_saved_file[ 'out.weight'].shape[0] del detect_saved_file del denoised_saved_file torch.cuda.empty_cache() if spike_size_nn_detector != spike_size_nn_denoiser: raise ValueError( 'input spike sizes of nn detector and denoiser do not match. change models' ) else: spike_size_nn = spike_size_nn_detector else: spike_size_nn = int( np.round(self.neuralnetwork.training.spike_size_ms * self.recordings.sampling_rate / 1000)) if spike_size_nn % 2 == 0: spike_size_nn += 1 self._set_param('spike_size_nn', spike_size_nn) else: self._set_param('spike_size_nn', center_spike_size) # torch devices devices = [] if torch.cuda.is_available(): n_processors = np.min( (torch.cuda.device_count(), self.resources.n_gpu_processors)) for j in range(n_processors): devices.append(torch.device("cuda:{}".format(j))) if len(devices) == 0: devices = [torch.device("cpu")] self._set_param('torch_devices', devices) # compute the length of recording filename_dat = os.path.join(self.data.root_folder, self.data.recordings) filesize = os.path.getsize(filename_dat) dtype = np.dtype(self.recordings.dtype) rec_len = int(filesize / dtype.itemsize / self.recordings.n_channels) self._set_param('rec_len', rec_len) # if self.recordings.final_deconv_chunk is None: start = 0 end = int(np.ceil(self.rec_len / self.recordings.sampling_rate)) else: start = int(np.floor(self.recordings.final_deconv_chunk[0])) end = int(np.ceil(self.recordings.final_deconv_chunk[1])) self._set_param('final_deconv_chunk', [start, end]) # if self.recordings.clustering_chunk is None: start = 0 end = int(np.ceil(self.rec_len / self.recordings.sampling_rate)) else: start = int(np.floor(self.recordings.clustering_chunk[0])) end = int(np.ceil(self.recordings.clustering_chunk[1])) self._set_param('clustering_chunk', [start, end])
def test_can_compute_whiten_matrix(data, data_info, path_to_geometry): geometry = parse(path_to_geometry, data_info['n_channels']) neighbors = find_channel_neighbors(geometry, radius=70) channel_index = make_channel_index(neighbors, geometry) whiten._matrix(data, channel_index, data_info['spike_size'])
def test_can_use_threshold_detector(data, data_info, path_to_geometry): geometry = parse(path_to_geometry, data_info['n_channels']) neighbors = find_channel_neighbors(geometry, radius=70) detect._threshold(data, neighbors, data_info['spike_size'], 5)
def test_can_compute_n_steps_neighbors(data_info, path_to_geometry): geometry = parse(path_to_geometry, data_info['n_channels']) neighbors = find_channel_neighbors(geometry, radius=70) n_steps_neigh_channels(neighbors, steps=2)
def test_can_compute_channel_neighbors(data_info, path_to_geometry): geometry = parse(path_to_geometry, data_info['n_channels']) find_channel_neighbors(geometry, radius=70)
def collision_rate(spike_train, templates, geometry, pair_wise=True, collision_threshold=10, neighborhood_distance=100): """ Parameters: ----------- spike_train: numpy.ndarray shape (N, 2) First column is spike time and second column unit id templates: numpy.ndarray shape (#units, #channels, #timesamples) geometry: numpy.ndarray shape (#channels, 2) pair_wise: bool If True, pairwise collision is reported. Otherwise, total collisions with other units is recorded. collision_threshold: int Threshold for spike time proximity to constitute a collision. neighborhood_distance: float Maximum distance of any two neighboring channels. Returns: -------- numpy.ndarray. If pair_wise is True, shape is (#units, #units). Otherwise shape is (#units,). """ def match(seq1, seq2): """Matches collisions of two spike time lists.""" m, n = len(seq1), len(seq2) i, j = 0, 0 res = 0 while i < m and j < n: if abs(seq1[i] - seq2[j]) < collision_threshold: res += 1 i += 1 elif seq1[i] < seq2[j]: i += 1 else: j += 1 return res # Main channel for each template main_chan = templates.ptp(axis=-1).argmax(axis=-1) chan_neighbors = find_channel_neighbors(geom, neighborhood_distance) n_unit = templates.shape[0] # Get spike train for each individual unit unit_spt = [spike_train[spike_train[:, 1] == i, 0] for i in range(n_unit)] # Main part of algorithm. if pair_wise: collisions = np.zeros([n_unit, n_unit]) for i in tqdm(range(n_unit)): spatial_radius = np.where(chan_neighbors[main_chan[i]])[0] for j in range(n_unit): if i == j or main_chan[j] not in spatial_radius: continue collisions[i, j] = match(unit_spt[i], unit_spt[j]) else: collisions = np.zeros([n_unit]) for i in tqdm(range(n_unit)): spatial_radius = np.where(chan_neighbors[main_chan[i]])[0] neighbor_units_spt = [] for j in range(n_unit): if i == j or main_chan[j] not in spatial_radius: continue neighbor_units_spt = merge(neighbor_units_spt, unit_spt[j]) collisions[i] = match(unit_spt[i], list(neighbor_units_spt)) return collisions
# path to 49 channels data root = os.path.expanduser('~/data/yass') path_to_data = os.path.join(root, 'ej49_data1_set1.bin') path_to_geom = os.path.join(root, 'ej49_geometry1.txt') observations = 6000000 n_channels = 49 sampling_freq = 20000 rec = np.fromfile(path_to_data, dtype='int16').reshape(observations, n_channels) rec.shape # TODO: check number of channels in neuropixel data geom = geometry.parse(path_to_geom, n_channels) neighbors = geometry.find_channel_neighbors(geom, radius=70) # get some observations from channel 0 raw_data = rec[50000:51000, 0] filtered = preprocess.butterworth(raw_data, low_freq=300, high_factor=0.1, order=3, sampling_freq=sampling_freq) standardized = preprocess.standarize(filtered, sampling_freq=sampling_freq) fix, (ax1, ax2, ax3) = plt.subplots(nrows=3) ax1.plot(raw_data) ax1.set_title('Raw')
def test_can_compute_whiten_matrix(data, path_to_geometry): geometry = parse(path_to_geometry, n_channels) neighbors = find_channel_neighbors(geometry, radius=70) whiten.matrix(data, neighbors, spike_size)
def test_can_whiten_data(data, path_to_geometry): geometry = parse(path_to_geometry, n_channels) neighbors = find_channel_neighbors(geometry, radius=70) q = whitening_matrix(data, neighbors, spike_size) whitening(data, q)
def test_can_use_threshold_detector(data, path_to_geometry): geometry = parse(path_to_geometry, n_channels) neighbors = find_channel_neighbors(geometry, radius=70) threshold_detection(data, neighbors, spike_size, 5)
def test_can_compute_n_steps_neighbors(path_to_geometry): geometry = parse(path_to_geometry, n_channels) neighbors = find_channel_neighbors(geometry, radius=70) n_steps_neigh_channels(neighbors, steps=2)
def test_can_compute_channel_neighbors(path_to_geometry): geometry = parse(path_to_geometry, n_channels) find_channel_neighbors(geometry, radius=70)