Example #1
0
def test_peeler_several_chunksize():

    dataio = DataIO(dirname='test_peeler')
    print(dataio)
    catalogue = dataio.load_catalogue(chan_grp=0)

    all_spikes = []
    sig_length = dataio.get_segment_length(0)
    chunksizes = [174, 512, 1024, 1023, 10000, 150000]
    #~ chunksizes = [512, 1024,]
    for chunksize in chunksizes:
        print('**', chunksize, '**')
        peeler = Peeler(dataio)
        peeler.change_params(engine='geometrical',
                             catalogue=catalogue,
                             chunksize=chunksize,
                             argmin_method='numba')
        t1 = time.perf_counter()
        peeler.run(progressbar=False)
        t2 = time.perf_counter()
        print('extra_size', peeler.peeler_engine.extra_size, 'n_span',
              peeler.peeler_engine.n_span, 'peak_width',
              peeler.peeler_engine.peak_width)
        print('peeler.run_loop', t2 - t1)

        # copy is need because the memmap is reset at each loop
        spikes = dataio.get_spikes(seg_num=0, chan_grp=0).copy()
        all_spikes.append(spikes)
        print(spikes.size)

    # clip to last spike
    last = min([spikes[-1]['index'] for spikes in all_spikes])
    for i, chunksize in enumerate(chunksizes):
        spikes = all_spikes[i]
        all_spikes[i] = spikes[spikes['index'] <= last]

    previsous_spikes = None
    for i, chunksize in enumerate(chunksizes):
        print('**', chunksize, '**')
        spikes = all_spikes[i]
        is_sorted = np.all(np.diff(spikes['index']) >= 0)
        assert is_sorted

        labeled_spike = spikes[spikes['cluster_label'] >= 0]
        unlabeled_spike = spikes[spikes['cluster_label'] < 0]
        print('labeled_spike.size', labeled_spike.size, 'unlabeled_spike.size',
              unlabeled_spike.size)
        print(spikes)

        # TODO: Peeler chunksize influence the number of spikes

        if previsous_spikes is not None:
            assert previsous_spikes.size == spikes.size
            np.testing.assert_array_equal(previsous_spikes['index'],
                                          spikes['index'])
            np.testing.assert_array_equal(previsous_spikes['cluster_label'],
                                          spikes['cluster_label'])

        previsous_spikes = spikes
Example #2
0
def test_peeler_several_chunksize():

    dataio = DataIO(dirname='test_peeler')
    print(dataio)
    catalogue = dataio.load_catalogue(chan_grp=0)

    all_spikes = []
    sig_length = dataio.get_segment_length(0)
    chunksizes = [174, 512, 1024, 1023, 10000, 150000]
    #~ chunksizes = [512, 1024,]
    for chunksize in chunksizes:
        print('**', chunksize, '**')
        peeler = Peeler(dataio)
        peeler.change_params(catalogue=catalogue, chunksize=chunksize)
        t1 = time.perf_counter()
        peeler.run_offline_loop_one_segment(seg_num=0, progressbar=False)
        t2 = time.perf_counter()
        print('n_side', peeler.n_side, 'n_span', peeler.n_span, 'peak_width',
              peeler.peak_width)
        print('peeler.run_loop', t2 - t1)

        spikes = dataio.get_spikes(seg_num=0, chan_grp=0)
        all_spikes.append(spikes)

    # clip to last spike
    last = min([spikes[-1]['index'] for spikes in all_spikes])
    for i, chunksize in enumerate(chunksizes):
        spikes = all_spikes[i]
        all_spikes[i] = spikes[spikes['index'] <= last]

    previsous_spikes = None
    for i, chunksize in enumerate(chunksizes):
        print('**', chunksize, '**')
        spikes = all_spikes[i]
        is_sorted = np.all(np.diff(spikes['index']) >= 0)
        assert is_sorted

        labeled_spike = spikes[spikes['cluster_label'] >= 0]
        unlabeled_spike = spikes[spikes['cluster_label'] < 0]
        print('labeled_spike.size', labeled_spike.size, 'unlabeled_spike.size',
              unlabeled_spike.size)

        if previsous_spikes is not None:
            assert previsous_spikes.size == spikes.size
            np.testing.assert_array_equal(previsous_spikes['index'],
                                          spikes['index'])
            np.testing.assert_array_equal(previsous_spikes['cluster_label'],
                                          spikes['cluster_label'])

        previsous_spikes = spikes
Example #3
0
def test_peeler_empty_catalogue():
    """
    This test peeler with empty catalogue.
    This is like a peak detector.
    Check several chunksize and compare to offline-one-buffer.
    
    """
    dataio = DataIO(dirname='test_peeler')
    #~ print(dataio)
    catalogue = dataio.load_catalogue(chan_grp=0)
    
    # empty catalogue for debug peak detection
    s = catalogue['centers0'].shape
    empty_centers = np.zeros((0, s[1], s[2]), dtype='float32')
    catalogue['centers0'] = empty_centers
    catalogue['centers1'] = empty_centers
    catalogue['centers2'] = empty_centers
    catalogue['cluster_labels'] = np.zeros(0, dtype=catalogue['cluster_labels'].dtype)
        
    
    sig_length = dataio.get_segment_length(0)
    chunksizes = [ 101, 174, 512, 1024, 1023, 10000, 150000]
    #~ chunksizes = [1024,]
    
    previous_peak = None
    
    for chunksize in chunksizes:
        print('**',  chunksize, '**')
        peeler = Peeler(dataio)
        peeler.change_params(engine='classic', catalogue=catalogue,chunksize=chunksize)
        t1 = time.perf_counter()
        #~ peeler.run(progressbar=False)
        peeler.run_offline_loop_one_segment(seg_num=0, progressbar=False)
        t2 = time.perf_counter()
        
        #~ print('n_side', peeler.n_side, 'n_span', peeler.n_span, 'peak_width', peeler.peak_width)
        #~ print('peeler.run_loop', t2-t1)
        
        spikes = dataio.get_spikes(seg_num=0, chan_grp=0)
        labeled_spike = spikes[spikes['cluster_label']>=0]
        unlabeled_spike = spikes[spikes['cluster_label']<0]
        assert labeled_spike.size == 0
        
        is_sorted = np.all(np.diff(unlabeled_spike['index'])>=0)
        assert is_sorted
        
        
        online_peaks = unlabeled_spike['index']
        engine = peeler.peeler_engine
        
        i_stop = sig_length-catalogue['signal_preprocessor_params']['lostfront_chunksize']-engine.n_side+engine.n_span
        sigs = dataio.get_signals_chunk(signal_type='processed', i_stop=i_stop)
        offline_peaks  = detect_peaks_in_chunk(sigs, engine.n_span, engine.relative_threshold, engine.peak_sign)
        
        offline_peaks  = offline_peaks[offline_peaks<=online_peaks[-1]]
        
        assert offline_peaks.size == online_peaks.size
        np.testing.assert_array_equal(offline_peaks, online_peaks)
        
        if previous_peak is not None:
            last = min(previous_peak[-1], online_peaks[-1])
            previous_peak = previous_peak[previous_peak<=last]
            online_peaks_cliped = online_peaks[online_peaks<=last]
            assert  previous_peak.size == online_peaks_cliped.size
            np.testing.assert_array_equal(previous_peak, online_peaks_cliped)
        
        previous_peak = online_peaks