def test_templates_loads_from_disk_if_files_exist(caplog, path_to_threshold_config): yass.set_config(path_to_threshold_config) (standarized_path, standarized_params, channel_index, whiten_filter) = preprocess.run() (score, spike_index_clear, spike_index_all) = detect.run(standarized_path, standarized_params, channel_index, whiten_filter) spike_train_clear, tmp_loc, vbParam = cluster.run( score, spike_index_clear) # save results templates.run(spike_train_clear, tmp_loc, save_results=True) assert templates.run.executed # next time this should not run and just load from files templates.run(spike_train_clear, tmp_loc, save_results=True) assert not templates.run.executed
def test_templates_returns_expected_results(path_to_config, path_to_output_reference, make_tmp_folder): yass.set_config(path_to_config, make_tmp_folder) (standarized_path, standarized_params, whiten_filter) = preprocess.run() (spike_index_clear, spike_index_all) = detect.run(standarized_path, standarized_params, whiten_filter) (spike_train_clear, tmp_loc, vbParam) = cluster.run(spike_index_clear) (templates_, spike_train, groups, idx_good_templates) = templates.run(spike_train_clear, tmp_loc, save_results=True) path_to_templates = path.join(path_to_output_reference, 'templates.npy') ReferenceTesting.assert_array_equal(templates_, path_to_templates)
def test_templates_returns_expected_results(path_to_threshold_config, path_to_data_folder): np.random.seed(0) yass.set_config(path_to_threshold_config) (standarized_path, standarized_params, channel_index, whiten_filter) = preprocess.run() (score, spike_index_clear, spike_index_all) = detect.run(standarized_path, standarized_params, channel_index, whiten_filter) spike_train_clear, tmp_loc, vbParam = cluster.run(score, spike_index_clear) (templates_, spike_train, groups, idx_good_templates) = templates.run(spike_train_clear, tmp_loc) path_to_templates = path.join(path_to_data_folder, 'output_reference', 'templates.npy') ReferenceTesting.assert_array_equal(templates_, path_to_templates) clean_tmp()
def test_templates(path_to_threshold_config): yass.set_config(path_to_threshold_config) (standarized_path, standarized_params, channel_index, whiten_filter) = preprocess.run() (score, spike_index_clear, spike_index_all) = detect.run(standarized_path, standarized_params, channel_index, whiten_filter) spike_train_clear, tmp_loc, vbParam = cluster.run( score, spike_index_clear) templates.run(spike_train_clear, tmp_loc) clean_tmp()
def main(): settings.run() start = datetime.now() logger = logging.getLogger(__name__) logger.info('Preprocessing started at second: %.2f', (datetime.now() - start).total_seconds()) # preprocessing (standarized_path, standarized_params, channel_index, whiten_filter) = preprocess.run(output_directory='profiling', if_file_exists='overwrite') logger.info('Preprocessing finished and detection started at second: %.2f', (datetime.now() - start).total_seconds()) # detection (score, spike_index_clear, spike_index_all) = detect.run(standarized_path, standarized_params, channel_index, whiten_filter, output_directory='profiling', if_file_exists='overwrite', save_results=True) logger.info('Detection finished and clustering started at second: %.2f', (datetime.now() - start).total_seconds()) # clustering spike_train_clear = cluster.run(score, spike_index_clear, output_directory='profiling', if_file_exists='overwrite', save_results=True) logger.info('Clustering finished and templates started at second: %.2f', (datetime.now() - start).total_seconds()) # templates the_templates = templates.run(spike_train_clear, output_directory='profiling', if_file_exists='overwrite', save_results=True) logger.info('templates finished and deconvolution started at second: %.2f', (datetime.now() - start).total_seconds()) # deconvolution deconvolute.run(spike_index_all, the_templates, output_directory='profiling') logger.info('Deconvolution finished at second: %.2f', (datetime.now() - start).total_seconds())
def test_decovnolution(path_to_threshold_config): yass.set_config('tests/config_nnet.yaml') (standarized_path, standarized_params, whiten_filter) = preprocess.run() (score, spike_index_clear, spike_index_all) = detect.run(standarized_path, standarized_params, whiten_filter) spike_train_clear, tmp_loc, vbParam = cluster.run(score, spike_index_clear) (templates_, spike_train, groups, idx_good_templates) = templates.run(spike_train_clear, tmp_loc) deconvolute.run(spike_index_all, templates_) clean_tmp()
def main(): settings.run() start = datetime.now() logger = logging.getLogger(__name__) logger.info('Preprocessing started at second: %.2f', (datetime.now() - start).total_seconds()) # preprocessing (standarized_path, standarized_params, channel_index, whiten_filter) = preprocess.run() logger.info('Preprocessing finished and detection started at second: %.2f', (datetime.now() - start).total_seconds()) # detection (score, spike_index_clear, spike_index_all) = detect.run(standarized_path, standarized_params, channel_index, whiten_filter) logger.info('Detection finished and clustering started at second: %.2f', (datetime.now() - start).total_seconds()) # clustering spike_train_clear = cluster.run(score, spike_index_clear) logger.info('Clustering finished and templates started at second: %.2f', (datetime.now() - start).total_seconds()) # templates the_templates = templates.run(spike_train_clear) logger.info('templates finished and deconvolution started at second: %.2f', (datetime.now() - start).total_seconds()) # deconvolution deconvolute.run(spike_index_all, the_templates) logger.info('Deconvolution finished at second: %.2f', (datetime.now() - start).total_seconds())
import numpy as np import logging import yass from yass import preprocess from yass import detect from yass import cluster from yass import templates np.random.seed(0) # configure logging module to get useful information logging.basicConfig(level=logging.INFO) # set yass configuration parameters yass.set_config('config_sample.yaml', 'templates-example') standarized_path, standarized_params, whiten_filter = preprocess.run() (spike_index_clear, spike_index_all) = detect.run(standarized_path, standarized_params, whiten_filter) spike_train_clear, tmp_loc, vbParam = cluster.run(spike_index_clear) (templates_, spike_train, groups, idx_good_templates) = templates.run(spike_train_clear, tmp_loc)
def run(config, logger_level='INFO', clean=False, output_dir='tmp/', complete=False, set_zero_seed=False): """Run YASS built-in pipeline Parameters ---------- config: str or mapping (such as dictionary) Path to YASS configuration file or mapping object logger_level: str Logger level clean: bool, optional Delete CONFIG.data.root_folder/output_dir/ before running output_dir: str, optional Output directory (relative to CONFIG.data.root_folder to store the output data, defaults to tmp/ complete: bool, optional Generates extra files (needed to generate phy files) Notes ----- Running the preprocessor will generate the followiing files in CONFIG.data.root_folder/output_directory/: * ``config.yaml`` - Copy of the configuration file * ``metadata.yaml`` - Experiment metadata * ``filtered.bin`` - Filtered recordings (from preprocess) * ``filtered.yaml`` - Filtered recordings metadata (from preprocess) * ``standarized.bin`` - Standarized recordings (from preprocess) * ``standarized.yaml`` - Standarized recordings metadata (from preprocess) * ``whitening.npy`` - Whitening filter (from preprocess) Returns ------- numpy.ndarray Spike train """ # load yass configuration parameters set_config(config) CONFIG = read_config() ROOT_FOLDER = CONFIG.data.root_folder TMP_FOLDER = path.join(ROOT_FOLDER, output_dir) # remove tmp folder if needed if os.path.exists(TMP_FOLDER) and clean: shutil.rmtree(TMP_FOLDER) # create TMP_FOLDER if needed if not os.path.exists(TMP_FOLDER): os.makedirs(TMP_FOLDER) # load logging config file logging_config = load_logging_config_file() logging_config['handlers']['file']['filename'] = path.join(TMP_FOLDER, 'yass.log') logging_config['root']['level'] = logger_level # configure logging logging.config.dictConfig(logging_config) # instantiate logger and start coloredlogs logger = logging.getLogger(__name__) coloredlogs.install(logger=logger) if set_zero_seed: logger.warning('Set numpy seed to zero') np.random.seed(0) # print yass version logger.info('YASS version: %s', yass.__version__) # preprocess start = time.time() (standarized_path, standarized_params, channel_index, whiten_filter) = (preprocess .run(output_directory=output_dir, if_file_exists=CONFIG.preprocess.if_file_exists)) time_preprocess = time.time() - start # detect start = time.time() (score, spike_index_clear, spike_index_all) = detect.run(standarized_path, standarized_params, channel_index, whiten_filter, output_directory=output_dir, if_file_exists=CONFIG.detect.if_file_exists, save_results=CONFIG.detect.save_results) time_detect = time.time() - start # cluster start = time.time() spike_train_clear, tmp_loc, vbParam = cluster.run( score, spike_index_clear, output_directory=output_dir, if_file_exists=CONFIG.cluster.if_file_exists, save_results=CONFIG.cluster.save_results) time_cluster = time.time() - start # get templates start = time.time() (templates, spike_train_clear_after_templates, groups, idx_good_templates) = get_templates.run( spike_train_clear, tmp_loc, output_directory=output_dir, if_file_exists=CONFIG.templates.if_file_exists, save_results=CONFIG.templates.save_results) time_templates = time.time() - start # run deconvolution start = time.time() spike_train = deconvolute.run(spike_index_all, templates, output_directory=output_dir) time_deconvolution = time.time() - start # save metadata in tmp path_to_metadata = path.join(TMP_FOLDER, 'metadata.yaml') logging.info('Saving metadata in {}'.format(path_to_metadata)) save_metadata(path_to_metadata) # save config.yaml copy in tmp/ path_to_config_copy = path.join(TMP_FOLDER, 'config.yaml') if isinstance(config, Mapping): with open(path_to_config_copy, 'w') as f: yaml.dump(config, f, default_flow_style=False) else: shutil.copy2(config, path_to_config_copy) logging.info('Saving copy of config: {} in {}'.format(config, path_to_config_copy)) # TODO: complete flag saves other files needed for integrating phy # with yass, the integration hasn't been completed yet # this part loads waveforms for all spikes in the spike train and scores # them, this data is needed to later generate phy files if complete: STANDARIZED_PATH = path.join(TMP_FOLDER, 'standarized.bin') PARAMS = load_yaml(path.join(TMP_FOLDER, 'standarized.yaml')) # load waveforms for all spikes in the spike train logger.info('Loading waveforms from all spikes in the spike train...') explorer = RecordingExplorer(STANDARIZED_PATH, spike_size=CONFIG.spike_size, dtype=PARAMS['dtype'], n_channels=PARAMS['n_channels'], data_order=PARAMS['data_order']) waveforms = explorer.read_waveforms(spike_train[:, 0]) path_to_waveforms = path.join(TMP_FOLDER, 'spike_train_waveforms.npy') np.save(path_to_waveforms, waveforms) logger.info('Saved all waveforms from the spike train in {}...' .format(path_to_waveforms)) # score all waveforms logger.info('Scoring waveforms from all spikes in the spike train...') path_to_rotation = path.join(TMP_FOLDER, 'rotation.npy') rotation = np.load(path_to_rotation) main_channels = explorer.main_channel_for_waveforms(waveforms) path_to_main_channels = path.join(TMP_FOLDER, 'waveforms_main_channel.npy') np.save(path_to_main_channels, main_channels) logger.info('Saved all waveforms main channels in {}...' .format(path_to_waveforms)) waveforms_score = dim_red.score(waveforms, rotation, main_channels, CONFIG.neigh_channels, CONFIG.geom) path_to_waveforms_score = path.join(TMP_FOLDER, 'waveforms_score.npy') np.save(path_to_waveforms_score, waveforms_score) logger.info('Saved all scores in {}...'.format(path_to_waveforms)) # score templates # TODO: templates should be returned in the right shape to avoid .T templates_ = templates.T main_channels_tmpls = explorer.main_channel_for_waveforms(templates_) path_to_templates_main_c = path.join(TMP_FOLDER, 'templates_main_channel.npy') np.save(path_to_templates_main_c, main_channels_tmpls) logger.info('Saved all templates main channels in {}...' .format(path_to_templates_main_c)) templates_score = dim_red.score(templates_, rotation, main_channels_tmpls, CONFIG.neigh_channels, CONFIG.geom) path_to_templates_score = path.join(TMP_FOLDER, 'templates_score.npy') np.save(path_to_templates_score, templates_score) logger.info('Saved all templates scores in {}...' .format(path_to_waveforms)) logger.info('Finished YASS execution. Timing summary:') total = (time_preprocess + time_detect + time_cluster + time_templates + time_deconvolution) logger.info('\t Preprocess: %s (%.2f %%)', human_readable_time(time_preprocess), time_preprocess/total*100) logger.info('\t Detection: %s (%.2f %%)', human_readable_time(time_detect), time_detect/total*100) logger.info('\t Clustering: %s (%.2f %%)', human_readable_time(time_cluster), time_cluster/total*100) logger.info('\t Templates: %s (%.2f %%)', human_readable_time(time_templates), time_templates/total*100) logger.info('\t Deconvolution: %s (%.2f %%)', human_readable_time(time_deconvolution), time_deconvolution/total*100) return spike_train
def test_nn_output(path_to_tests): """Test that pipeline using threshold detector returns the same results """ logger = logging.getLogger(__name__) yass.set_config(path.join(path_to_tests, 'config_nn_49.yaml')) CONFIG = read_config() TMP = Path(CONFIG.data.root_folder, 'tmp') logger.info('Removing %s', TMP) shutil.rmtree(str(TMP)) PATH_TO_REF = '/home/Edu/data/nnet' np.random.seed(0) # run preprocess (standarized_path, standarized_params, whiten_filter) = preprocess.run() # load preprocess output path_to_standarized = path.join(PATH_TO_REF, 'preprocess', 'standarized.bin') path_to_whitening = path.join(PATH_TO_REF, 'preprocess', 'whitening.npy') whitening_saved = np.load(path_to_whitening) standarized_saved = RecordingsReader(path_to_standarized, loader='array').data standarized = RecordingsReader(standarized_path, loader='array').data # test preprocess np.testing.assert_array_equal(whitening_saved, whiten_filter) np.testing.assert_array_equal(standarized_saved, standarized) # run detect (score, spike_index_clear, spike_index_all) = detect.run(standarized_path, standarized_params, whiten_filter) # load detect output path_to_scores = path.join(PATH_TO_REF, 'detect', 'scores_clear.npy') path_to_spike_index_clear = path.join(PATH_TO_REF, 'detect', 'spike_index_clear.npy') path_to_spike_index_all = path.join(PATH_TO_REF, 'detect', 'spike_index_all.npy') scores_saved = np.load(path_to_scores) spike_index_clear_saved = np.load(path_to_spike_index_clear) spike_index_all_saved = np.load(path_to_spike_index_all) # test detect output np.testing.assert_array_equal(scores_saved, score) np.testing.assert_array_equal(spike_index_clear_saved, spike_index_clear) np.testing.assert_array_equal(spike_index_all_saved, spike_index_all) # run cluster (spike_train_clear, tmp_loc, vbParam) = cluster.run(score, spike_index_clear) # load cluster output path_to_spike_train_cluster = path.join(PATH_TO_REF, 'cluster', 'spike_train_cluster.npy') spike_train_cluster_saved = np.load(path_to_spike_train_cluster) # test cluster #np.testing.assert_array_equal(spike_train_cluster_saved, spike_train_clear) # run templates (templates_, spike_train, groups, idx_good_templates) = templates.run(spike_train_clear, tmp_loc, save_results=True) # load templates output path_to_templates = path.join(PATH_TO_REF, 'templates', 'templates.npy') templates_saved = np.load(path_to_templates) # test templates np.testing.assert_array_almost_equal(templates_saved, templates_, decimal=4) # run deconvolution spike_train = deconvolute.run(spike_index_all, templates_) # load deconvolution output path_to_spike_train = path.join(PATH_TO_REF, 'spike_train.npy') spike_train_saved = np.load(path_to_spike_train) # test deconvolution np.testing.assert_array_equal(spike_train_saved, spike_train)