def _neural_network_detection(standarized_path, standarized_params,
n_observations, output_directory):
"""Run neural network detection and autoencoder dimensionality reduction
"""
logger = logging.getLogger(__name__)
CONFIG = read_config()
OUTPUT_DTYPE = CONFIG.preprocess.dtype
TMP_FOLDER = os.path.join(CONFIG.data.root_folder, output_directory)
# detect spikes
bp = BatchProcessor(standarized_path,
standarized_params['dtype'],
standarized_params['n_channels'],
standarized_params['data_format'],
CONFIG.resources.max_memory,
buffer_size=0)
# check if all scores, clear and collision spikes exist..
path_to_score = os.path.join(TMP_FOLDER, 'score_clear.npy')
path_to_spike_index_clear = os.path.join(TMP_FOLDER,
'spike_index_clear.npy')
path_to_spike_index_collision = os.path.join(TMP_FOLDER,
'spike_index_collision.npy')
if all([
os.path.exists(path_to_score),
os.path.exists(path_to_spike_index_clear),
os.path.exists(path_to_spike_index_collision)
]):
logger.info('Loading "{}", "{}" and "{}"'.format(
path_to_score, path_to_spike_index_clear,
path_to_spike_index_collision))
scores = np.load(path_to_score)
clear = np.load(path_to_spike_index_clear)
collision = np.load(path_to_spike_index_collision)
else:
logger.info('One or more of "{}", "{}" or "{}" files were missing, '
'computing...'.format(path_to_score,
path_to_spike_index_clear,
path_to_spike_index_collision))
# apply threshold detector on standarized data
autoencoder_filename = CONFIG.neural_network_autoencoder.filename
mc = bp.multi_channel_apply
res = mc(
neuralnetwork.nn_detection,
mode='memory',
cleanup_function=neuralnetwork.fix_indexes,
neighbors=CONFIG.neighChannels,
geom=CONFIG.geom,
temporal_features=CONFIG.spikes.temporal_features,
# FIXME: what is this?
temporal_window=3,
th_detect=CONFIG.neural_network_detector.threshold_spike,
th_triage=CONFIG.neural_network_triage.threshold_collision,
detector_filename=CONFIG.neural_network_detector.filename,
autoencoder_filename=autoencoder_filename,
triage_filename=CONFIG.neural_network_triage.filename)
# save clear spikes
clear = np.concatenate([element[1] for element in res], axis=0)
logger.info('Removing clear indexes outside the allowed range to '
'draw a complete waveform...')
clear, idx = detect.remove_incomplete_waveforms(
clear, CONFIG.spikeSize + CONFIG.templatesMaxShift, n_observations)
np.save(path_to_spike_index_clear, clear)
logger.info('Saved spike index clear in {}...'.format(
path_to_spike_index_clear))
# save collided spikes
collision = np.concatenate([element[2] for element in res], axis=0)
logger.info('Removing collision indexes outside the allowed range to '
'draw a complete waveform...')
collision, _ = detect.remove_incomplete_waveforms(
collision, CONFIG.spikeSize + CONFIG.templatesMaxShift,
n_observations)
np.save(path_to_spike_index_collision, collision)
logger.info('Saved spike index collision in {}...'.format(
path_to_spike_index_collision))
if CONFIG.clustering.clustering_method == 'location':
#######################
# Waveform extraction #
#######################
# TODO: what should the behaviour be for spike indexes that are
# when starting/ending the recordings and it is not possible to
# draw a complete waveform?
logger.info('Computing whitening matrix...')
bp = BatchProcessor(standarized_path, standarized_params['dtype'],
standarized_params['n_channels'],
standarized_params['data_format'],
CONFIG.resources.max_memory)
batches = bp.multi_channel()
first_batch, _, _ = next(batches)
Q = whiten.matrix(first_batch, CONFIG.neighChannels,
CONFIG.spikeSize)
path_to_whitening_matrix = os.path.join(TMP_FOLDER,
'whitening.npy')
np.save(path_to_whitening_matrix, Q)
logger.info('Saved whitening matrix in {}'.format(
path_to_whitening_matrix))
# apply whitening to every batch
(whitened_path, whitened_params) = bp.multi_channel_apply(
np.matmul,
mode='disk',
output_path=os.path.join(TMP_FOLDER, 'whitened.bin'),
if_file_exists='skip',
cast_dtype=OUTPUT_DTYPE,
b=Q)
main_channel = clear[:, 1]
# load and dump waveforms from clear spikes
path_to_waveforms_clear = os.path.join(TMP_FOLDER,
'waveforms_clear.npy')
if os.path.exists(path_to_waveforms_clear):
logger.info(
'Found clear waveforms in {}, loading them...'.format(
path_to_waveforms_clear))
waveforms_clear = np.load(path_to_waveforms_clear)
else:
logger.info(
'Did not find clear waveforms in {}, reading them from {}'.
format(path_to_waveforms_clear, whitened_path))
explorer = RecordingExplorer(whitened_path,
spike_size=CONFIG.spikeSize)
waveforms_clear = explorer.read_waveforms(clear[:, 0], 'all')
np.save(path_to_waveforms_clear, waveforms_clear)
logger.info('Saved waveform from clear spikes in: {}'.format(
path_to_waveforms_clear))
main_channel = clear[:, 1]
# save rotation
detector_filename = CONFIG.neural_network_detector.filename
autoencoder_filename = CONFIG.neural_network_autoencoder.filename
rotation = neuralnetwork.load_rotation(detector_filename,
autoencoder_filename)
path_to_rotation = os.path.join(TMP_FOLDER, 'rotation.npy')
logger.info("rotation_matrix_shape = {}".format(rotation.shape))
np.save(path_to_rotation, rotation)
logger.info(
'Saved rotation matrix in {}...'.format(path_to_rotation))
logger.info('Denoising...')
path_to_denoised_waveforms = os.path.join(
TMP_FOLDER, 'denoised_waveforms.npy')
if os.path.exists(path_to_denoised_waveforms):
logger.info(
'Found denoised waveforms in {}, loading them...'.format(
path_to_denoised_waveforms))
denoised_waveforms = np.load(path_to_denoised_waveforms)
else:
logger.info(
'Did not find denoised waveforms in {}, evaluating them'
'from {}'.format(path_to_denoised_waveforms,
path_to_waveforms_clear))
waveforms_clear = np.load(path_to_waveforms_clear)
denoised_waveforms = dim_red.denoise(waveforms_clear, rotation,
CONFIG)
logger.info('Saving denoised waveforms to {}'.format(
path_to_denoised_waveforms))
np.save(path_to_denoised_waveforms, denoised_waveforms)
isolated_index, x, y = get_isolated_spikes_and_locations(
denoised_waveforms, main_channel, CONFIG)
x = (x - np.mean(x)) / np.std(x)
y = (y - np.mean(y)) / np.std(y)
corrupted_index = np.logical_not(
np.in1d(np.arange(clear.shape[0]), isolated_index))
collision = np.concatenate([collision, clear[corrupted_index]],
axis=0)
clear = clear[isolated_index]
waveforms_clear = waveforms_clear[isolated_index]
#################################################
# Dimensionality reduction (Isolated Waveforms) #
#################################################
scores = dim_red.main_channel_scores(waveforms_clear, rotation,
clear, CONFIG)
scores = (scores - np.mean(scores, axis=0)) / np.std(scores)
scores = np.concatenate([
x[:, np.newaxis, np.newaxis], y[:, np.newaxis, np.newaxis],
scores[:, :, np.newaxis]
],
axis=1)
else:
# save scores
scores = np.concatenate([element[0] for element in res], axis=0)
logger.info(
'Removing scores for indexes outside the allowed range to '
'draw a complete waveform...')
scores = scores[idx]
# compute Q for whitening
logger.info('Computing whitening matrix...')
bp = BatchProcessor(standarized_path, standarized_params['dtype'],
standarized_params['n_channels'],
standarized_params['data_format'],
CONFIG.resources.max_memory)
batches = bp.multi_channel()
first_batch, _, _ = next(batches)
Q = whiten.matrix_localized(first_batch, CONFIG.neighChannels,
CONFIG.geom, CONFIG.spikeSize)
path_to_whitening_matrix = os.path.join(TMP_FOLDER,
'whitening.npy')
np.save(path_to_whitening_matrix, Q)
logger.info('Saved whitening matrix in {}'.format(
path_to_whitening_matrix))
scores = whiten.score(scores, clear[:, 1], Q)
np.save(path_to_score, scores)
logger.info('Saved spike scores in {}...'.format(path_to_score))
# save rotation
detector_filename = CONFIG.neural_network_detector.filename
autoencoder_filename = CONFIG.neural_network_autoencoder.filename
rotation = neuralnetwork.load_rotation(detector_filename,
autoencoder_filename)
path_to_rotation = os.path.join(TMP_FOLDER, 'rotation.npy')
np.save(path_to_rotation, rotation)
logger.info(
'Saved rotation matrix in {}...'.format(path_to_rotation))
np.save(path_to_score, scores)
logger.info('Saved spike scores in {}...'.format(path_to_score))
return scores, clear, collision
def run(output_directory='tmp/'):
"""Execute preprocessing pipeline
Parameters
----------
output_directory: str, optional
Location to store partial results, relative to CONFIG.data.root_folder,
defaults to tmp/
Returns
-------
clear_scores: numpy.ndarray (n_spikes, n_features, n_channels)
3D array with the scores for the clear spikes, first simension is
the number of spikes, second is the nymber of features and third the
number of channels
spike_index_clear: numpy.ndarray (n_clear_spikes, 2)
2D array with indexes for clear spikes, first column contains the
spike location in the recording and the second the main channel
(channel whose amplitude is maximum)
spike_index_collision: numpy.ndarray (n_collided_spikes, 2)
2D array with indexes for collided spikes, first column contains the
spike location in the recording and the second the main channel
(channel whose amplitude is maximum)
Notes
-----
Running the preprocessor will generate the followiing files in
CONFIG.data.root_folder/output_directory/:
* ``config.yaml`` - Copy of the configuration file
* ``metadata.yaml`` - Experiment metadata
* ``filtered.bin`` - Filtered recordings
* ``filtered.yaml`` - Filtered recordings metadata
* ``standarized.bin`` - Standarized recordings
* ``standarized.yaml`` - Standarized recordings metadata
* ``whitened.bin`` - Whitened recordings
* ``whitened.yaml`` - Whitened recordings metadata
* ``rotation.npy`` - Rotation matrix for dimensionality reduction
* ``spike_index_clear.npy`` - Same as spike_index_clear returned
* ``spike_index_collision.npy`` - Same as spike_index_collision returned
* ``score_clear.npy`` - Scores for clear spikes
* ``waveforms_clear.npy`` - Waveforms for clear spikes
Examples
--------
.. literalinclude:: ../examples/preprocess.py
"""
logger = logging.getLogger(__name__)
CONFIG = read_config()
OUTPUT_DTYPE = CONFIG.preprocess.dtype
logger.info(
'Output dtype for transformed data will be {}'.format(OUTPUT_DTYPE))
TMP = os.path.join(CONFIG.data.root_folder, output_directory)
if not os.path.exists(TMP):
logger.info('Creating temporary folder: {}'.format(TMP))
os.makedirs(TMP)
else:
logger.info('Temporary folder {} already exists, output will be '
'stored there'.format(TMP))
path = os.path.join(CONFIG.data.root_folder, CONFIG.data.recordings)
dtype = CONFIG.recordings.dtype
# initialize pipeline object, one batch per channel
pipeline = BatchPipeline(path, dtype, CONFIG.recordings.n_channels,
CONFIG.recordings.format,
CONFIG.resources.max_memory, TMP)
# add filter transformation if necessary
if CONFIG.preprocess.filter:
filter_op = Transform(butterworth,
'filtered.bin',
mode='single_channel_one_batch',
keep=True,
if_file_exists='skip',
cast_dtype=OUTPUT_DTYPE,
low_freq=CONFIG.filter.low_pass_freq,
high_factor=CONFIG.filter.high_factor,
order=CONFIG.filter.order,
sampling_freq=CONFIG.recordings.sampling_rate)
pipeline.add([filter_op])
(filtered_path, ), (filtered_params, ) = pipeline.run()
# standarize
bp = BatchProcessor(filtered_path, filtered_params['dtype'],
filtered_params['n_channels'],
filtered_params['data_format'],
CONFIG.resources.max_memory)
batches = bp.multi_channel()
first_batch, _, _ = next(batches)
sd = standard_deviation(first_batch, CONFIG.recordings.sampling_rate)
(standarized_path, standarized_params) = bp.multi_channel_apply(
standarize,
mode='disk',
output_path=os.path.join(TMP, 'standarized.bin'),
if_file_exists='skip',
cast_dtype=OUTPUT_DTYPE,
sd=sd)
standarized = RecordingsReader(standarized_path)
n_observations = standarized.observations
if CONFIG.spikes.detection == 'threshold':
return _threshold_detection(standarized_path, standarized_params,
n_observations, output_directory)
elif CONFIG.spikes.detection == 'nn':
return _neural_network_detection(standarized_path, standarized_params,
n_observations, output_directory)
def matrix(path_to_data,
dtype,
n_channels,
data_order,
channel_index,
spike_size,
max_memory,
output_path,
output_filename='whitening.npy',
if_file_exists='skip'):
"""Compute whitening filter using the first batch of the data
Parameters
----------
path_to_data: str
Path to recordings in binary format
dtype: str
Recordings dtype
n_channels: int
Number of channels in the recordings
data_order: str
Recordings order, one of ('channels', 'samples'). In a dataset with k
observations per channel and j channels: 'channels' means first k
contiguous observations come from channel 0, then channel 1, and so
on. 'sample' means first j contiguous data are the first observations
from all channels, then the second observations from all channels and
so on
channel_index: np.array
A matrix of size [n_channels, n_nieghbors], showing neighboring channel
information
spike_size: int
Spike size
max_memory: str
Max memory to use in each batch (e.g. 100MB, 1GB)
output_path: str
Where to store the whitenint gilter
output_filename: str, optional
Filename for the output data, defaults to whitening.npy
if_file_exists: str, optional
One of 'overwrite', 'abort', 'skip'. If 'overwrite' it replaces the
whitening filter if it exists, if 'abort' if raise a ValueError
exception if the file exists, if 'skip' if skips the operation if the
file exists
Returns
-------
standarized_path: str
Path to standarized recordings
standarized_params: dict
A dictionary with the parameters for the standarized recordings
(dtype, n_channels, data_order)
"""
logger = logging.getLogger(__name__)
# compute Q (using the first batchfor whitening
logger.info('Computing whitening matrix...')
bp = BatchProcessor(path_to_data, dtype, n_channels, data_order,
max_memory)
batches = bp.multi_channel()
first_batch = next(batches)
whiten_filter = _matrix(first_batch, channel_index, spike_size)
path_to_whitening_matrix = Path(output_path, output_filename)
save_numpy_object(whiten_filter,
path_to_whitening_matrix,
if_file_exists='overwrite',
name='whitening filter')
return whiten_filter
def _threshold_detection(standarized_path, standarized_params, n_observations,
output_directory):
"""Run threshold detector and dimensionality reduction using PCA
"""
logger = logging.getLogger(__name__)
CONFIG = read_config()
OUTPUT_DTYPE = CONFIG.preprocess.dtype
TMP_FOLDER = os.path.join(CONFIG.data.root_folder, output_directory)
###############
# Whiten data #
###############
# compute Q for whitening
logger.info('Computing whitening matrix...')
bp = BatchProcessor(standarized_path, standarized_params['dtype'],
standarized_params['n_channels'],
standarized_params['data_format'],
CONFIG.resources.max_memory)
batches = bp.multi_channel()
first_batch, _, _ = next(batches)
Q = whiten.matrix(first_batch, CONFIG.neighChannels, CONFIG.spikeSize)
path_to_whitening_matrix = os.path.join(TMP_FOLDER, 'whitening.npy')
np.save(path_to_whitening_matrix, Q)
logger.info(
'Saved whitening matrix in {}'.format(path_to_whitening_matrix))
# apply whitening to every batch
(whitened_path, whitened_params) = bp.multi_channel_apply(
np.matmul,
mode='disk',
output_path=os.path.join(TMP_FOLDER, 'whitened.bin'),
if_file_exists='skip',
cast_dtype=OUTPUT_DTYPE,
b=Q)
###################
# Spike detection #
###################
path_to_spike_index_clear = os.path.join(TMP_FOLDER,
'spike_index_clear.npy')
bp = BatchProcessor(standarized_path,
standarized_params['dtype'],
standarized_params['n_channels'],
standarized_params['data_format'],
CONFIG.resources.max_memory,
buffer_size=0)
# clear spikes
if os.path.exists(path_to_spike_index_clear):
# if it exists, load it...
logger.info('Found file in {}, loading it...'.format(
path_to_spike_index_clear))
spike_index_clear = np.load(path_to_spike_index_clear)
else:
# if it doesn't, detect spikes...
logger.info('Did not find file in {}, finding spikes using threshold'
' detector...'.format(path_to_spike_index_clear))
# apply threshold detector on standarized data
spikes = bp.multi_channel_apply(detect.threshold,
mode='memory',
cleanup_function=detect.fix_indexes,
neighbors=CONFIG.neighChannels,
spike_size=CONFIG.spikeSize,
std_factor=CONFIG.stdFactor)
spike_index_clear = np.vstack(spikes)
logger.info('Removing clear indexes outside the allowed range to '
'draw a complete waveform...')
spike_index_clear, _ = (detect.remove_incomplete_waveforms(
spike_index_clear, CONFIG.spikeSize + CONFIG.templatesMaxShift,
n_observations))
logger.info('Saving spikes in {}...'.format(path_to_spike_index_clear))
np.save(path_to_spike_index_clear, spike_index_clear)
path_to_spike_index_collision = os.path.join(TMP_FOLDER,
'spike_index_collision.npy')
# collided spikes
if os.path.exists(path_to_spike_index_collision):
# if it exists, load it...
logger.info('Found collided spikes in {}, loading them...'.format(
path_to_spike_index_collision))
spike_index_collision = np.load(path_to_spike_index_collision)
if spike_index_collision.shape[0] != 0:
raise ValueError('Found non-empty collision spike index in {}, '
'but threshold detector is selected, collision '
'detection is not implemented for threshold '
'detector so array must have dimensios (0, 2) '
'but had ({}, {})'.format(
path_to_spike_index_collision,
*spike_index_collision.shape))
else:
# triage is not implemented on threshold detector, return empty array
logger.info('Creating empty array for'
' collided spikes (collision detection is not implemented'
' with threshold detector. Saving them in {}'.format(
path_to_spike_index_collision))
spike_index_collision = np.zeros((0, 2), 'int32')
np.save(path_to_spike_index_collision, spike_index_collision)
#######################
# Waveform extraction #
#######################
# load and dump waveforms from clear spikes
path_to_waveforms_clear = os.path.join(TMP_FOLDER, 'waveforms_clear.npy')
if os.path.exists(path_to_waveforms_clear):
logger.info('Found clear waveforms in {}, loading them...'.format(
path_to_waveforms_clear))
waveforms_clear = np.load(path_to_waveforms_clear)
else:
logger.info(
'Did not find clear waveforms in {}, reading them from {}'.format(
path_to_waveforms_clear, standarized_path))
explorer = RecordingExplorer(standarized_path,
spike_size=CONFIG.spikeSize)
waveforms_clear = explorer.read_waveforms(spike_index_clear[:, 0])
np.save(path_to_waveforms_clear, waveforms_clear)
logger.info('Saved waveform from clear spikes in: {}'.format(
path_to_waveforms_clear))
#########################
# PCA - rotation matrix #
#########################
# compute per-batch sufficient statistics for PCA on standarized data
logger.info('Computing PCA sufficient statistics...')
stats = bp.multi_channel_apply(dim_red.suff_stat,
mode='memory',
spike_index=spike_index_clear,
spike_size=CONFIG.spikeSize)
suff_stats = reduce(lambda x, y: np.add(x, y), [e[0] for e in stats])
spikes_per_channel = reduce(lambda x, y: np.add(x, y),
[e[1] for e in stats])
# compute rotation matrix
logger.info('Computing PCA projection matrix...')
rotation = dim_red.project(suff_stats, spikes_per_channel,
CONFIG.spikes.temporal_features,
CONFIG.neighChannels)
path_to_rotation = os.path.join(TMP_FOLDER, 'rotation.npy')
np.save(path_to_rotation, rotation)
logger.info('Saved rotation matrix in {}...'.format(path_to_rotation))
main_channel = spike_index_clear[:, 1]
###########################################
# PCA - waveform dimensionality reduction #
###########################################
if CONFIG.clustering.clustering_method == 'location':
logger.info('Denoising...')
path_to_denoised_waveforms = os.path.join(TMP_FOLDER,
'denoised_waveforms.npy')
if os.path.exists(path_to_denoised_waveforms):
logger.info(
'Found denoised waveforms in {}, loading them...'.format(
path_to_denoised_waveforms))
denoised_waveforms = np.load(path_to_denoised_waveforms)
else:
logger.info(
'Did not find denoised waveforms in {}, evaluating them'
'from {}'.format(path_to_denoised_waveforms,
path_to_waveforms_clear))
waveforms_clear = np.load(path_to_waveforms_clear)
denoised_waveforms = dim_red.denoise(waveforms_clear, rotation,
CONFIG)
logger.info('Saving denoised waveforms to {}'.format(
path_to_denoised_waveforms))
np.save(path_to_denoised_waveforms, denoised_waveforms)
isolated_index, x, y = get_isolated_spikes_and_locations(
denoised_waveforms, main_channel, CONFIG)
x = (x - np.mean(x)) / np.std(x)
y = (y - np.mean(y)) / np.std(y)
corrupted_index = np.logical_not(
np.in1d(np.arange(spike_index_clear.shape[0]), isolated_index))
spike_index_collision = np.concatenate(
[spike_index_collision, spike_index_clear[corrupted_index]],
axis=0)
spike_index_clear = spike_index_clear[isolated_index]
waveforms_clear = waveforms_clear[isolated_index]
#################################################
# Dimensionality reduction (Isolated Waveforms) #
#################################################
scores = dim_red.main_channel_scores(waveforms_clear, rotation,
spike_index_clear, CONFIG)
scores = (scores - np.mean(scores, axis=0)) / np.std(scores)
scores = np.concatenate([
x[:, np.newaxis, np.newaxis], y[:, np.newaxis, np.newaxis],
scores[:, :, np.newaxis]
],
axis=1)
else:
logger.info('Reducing spikes dimensionality with PCA matrix...')
scores = dim_red.score(waveforms_clear, rotation, spike_index_clear[:,
1],
CONFIG.neighChannels, CONFIG.geom)
# save scores
path_to_score = os.path.join(TMP_FOLDER, 'score_clear.npy')
np.save(path_to_score, scores)
logger.info('Saved spike scores in {}...'.format(path_to_score))
return scores, spike_index_clear, spike_index_collision
from yass.batch import BatchProcessor
path_to_neuropixel_data = (os.path.expanduser('~/data/ucl-neuropixel'
'/rawDataSample.bin'))
bp = BatchProcessor(path_to_neuropixel_data,
dtype='int16',
n_channels=385,
data_format='wide',
max_memory='300MB')
# now, let's to some multi_channel operations, here we will
# traverse all channels and all observations, each batch will
# contain a subset in the temporal dimension, the window size
# is determined by max_memory
data = bp.multi_channel()
for d, _, idx in data:
print('Shape: {}. Index: {}'.format(d.shape, idx))
# we can specify the temporal limits and subset channels
data = bp.multi_channel(from_time=100000, to_time=200000, channels=[0, 1, 2])
for d, _, idx in data:
print('Shape: {}. Index: {}'.format(d.shape, idx))
# we can also create a BatchProcessor with a buffer
bp2 = BatchProcessor(path_to_neuropixel_data,
dtype='int16',
n_channels=385,
data_format='wide',