def _run(self, recording, output_folder):
nb_chan = recording.get_num_channels()
# check params and OpenCL when many channels
use_sparse_template = False
use_opencl_with_sparse = False
if nb_chan >64: # this limit depend on the platform of course
if tdc.cltools.HAVE_PYOPENCL:
# force opencl
self.params['fullchain_kargs']['preprocessor']['signalpreprocessor_engine'] = 'opencl'
use_sparse_template = True
use_opencl_with_sparse = True
else:
print('OpenCL is not available processing will be slow, try install it')
tdc_dataio = tdc.DataIO(dirname=str(output_folder))
# make catalogue
nested_params = make_nested_tdc_params(**self.params)
chan_grps = list(tdc_dataio.channel_groups.keys())
for chan_grp in chan_grps:
cc = tdc.CatalogueConstructor(dataio=tdc_dataio, chan_grp=chan_grp)
tdc.apply_all_catalogue_steps(cc, verbose=self.debug, **nested_params)
if self.debug:
print(cc)
cc.make_catalogue_for_peeler()
# apply Peeler (template matching)
initial_catalogue = tdc_dataio.load_catalogue(chan_grp=chan_grp)
peeler = tdc.Peeler(tdc_dataio)
peeler.change_params(catalogue=initial_catalogue,
use_sparse_template=use_sparse_template,
sparse_threshold_mad=1.5,
use_opencl_with_sparse=use_opencl_with_sparse,)
peeler.run(duration=None, progressbar=self.debug)
def get_catalogueconstructor(self, channel_rel, label):
if not self.check_finished_loading():
return
label = 'ch{}'.format(label)
# Get absolute row index of data array.
channel_abs = self.dataio.datasource.channel_names.index(label)
channel_groups = {
label: {
'channels': [channel_abs],
'geometry': {
channel_abs: self.geometry[channel_rel]
}
}
}
path_probe = os.path.join(self.output_path, 'electrode_selection.prb')
with open(path_probe, 'w') as f:
f.write("channel_groups = {}".format(channel_groups))
self.dataio.set_probe_file(path_probe)
cc = tdc.CatalogueConstructor(self.dataio, cbnu=self)
cc.info.update(self.config)
cc.flush_info()
self._cc[len(self._cc)] = cc
return cc
def _run(self, recording, output_folder):
recording = recover_recording(recording)
tdc_dataio = tdc.DataIO(dirname=str(output_folder))
params = dict(self.params)
del params["chunk_mb"], params["n_jobs_bin"]
clean_catalogue_gui = params.pop('clean_catalogue_gui')
# make catalogue
chan_grps = list(tdc_dataio.channel_groups.keys())
for chan_grp in chan_grps:
# parameters can change depending the group
catalogue_nested_params = make_nested_tdc_params(
tdc_dataio, chan_grp, **params)
if self.verbose:
print('catalogue_nested_params')
pprint(catalogue_nested_params)
peeler_params = tdc.get_auto_params_for_peelers(
tdc_dataio, chan_grp)
if self.verbose:
print('peeler_params')
pprint(peeler_params)
cc = tdc.CatalogueConstructor(dataio=tdc_dataio, chan_grp=chan_grp)
tdc.apply_all_catalogue_steps(cc,
catalogue_nested_params,
verbose=self.verbose)
if clean_catalogue_gui:
import pyqtgraph as pg
app = pg.mkQApp()
win = tdc.CatalogueWindow(cc)
win.show()
app.exec_()
if self.verbose:
print(cc)
if distutils.version.LooseVersion(tdc.__version__) < '1.6.0':
print('You should upgrade tridesclous')
t0 = time.perf_counter()
cc.make_catalogue_for_peeler()
if self.verbose:
t1 = time.perf_counter()
print('make_catalogue_for_peeler', t1 - t0)
# apply Peeler (template matching)
initial_catalogue = tdc_dataio.load_catalogue(chan_grp=chan_grp)
peeler = tdc.Peeler(tdc_dataio)
peeler.change_params(catalogue=initial_catalogue, **peeler_params)
t0 = time.perf_counter()
peeler.run(duration=None, progressbar=False)
if self.verbose:
t1 = time.perf_counter()
print('peeler.tun', t1 - t0)
def run_all_catalogues():
files = pd.read_excel(basedir + 'file_list.xlsx')
group_names = np.unique(files['group_name'])
for group_name in group_names:
print('*******')
print(group_name)
dirname = tdc_workdir + group_name
dataio = tdc.DataIO(dirname=dirname)
print(dataio)
for chan_grp in dataio.channel_groups.keys():
cc = tdc.CatalogueConstructor(dataio=dataio, chan_grp=chan_grp)
print(cc)
fullchain_kargs = {
'duration': 300.,
'preprocessor': {
'highpass_freq': 400.,
'lowpass_freq': 5000.,
'smooth_size': 0,
'chunksize': 1024,
'lostfront_chunksize': 128,
'signalpreprocessor_engine': 'numpy',
},
'peak_detector': {
'peakdetector_engine': 'numpy',
'peak_sign': '-',
'relative_threshold': 4.5,
'peak_span': 0.0002,
},
'noise_snippet': {
'nb_snippet': 300,
},
'extract_waveforms': {
'n_left': -20,
'n_right': 30,
'mode': 'rand',
'nb_max': 20000,
'align_waveform': False,
},
'clean_waveforms': {
'alien_value_threshold': 25.,
},
}
feat_method = 'global_pca'
feat_kargs = {}
clust_method = 'sawchaincut'
clust_kargs = {}
tdc.apply_all_catalogue_steps(cc, fullchain_kargs, feat_method,
feat_kargs, clust_method,
clust_kargs)
print(cc)
def _run(self, recording, output_folder):
nb_chan = recording.get_num_channels()
tdc_dataio = tdc.DataIO(dirname=str(output_folder))
params = dict(self.params)
clean_catalogue_gui = params.pop('clean_catalogue_gui')
# make catalogue
chan_grps = list(tdc_dataio.channel_groups.keys())
for chan_grp in chan_grps:
# parameters can change depending the group
catalogue_nested_params = make_nested_tdc_params(tdc_dataio, chan_grp, **params)
if self.verbose:
print('catalogue_nested_params')
pprint(catalogue_nested_params)
peeler_params = tdc.get_auto_params_for_peelers(tdc_dataio, chan_grp)
if self.verbose:
print('peeler_params')
pprint(peeler_params)
# check params and OpenCL when many channels
use_sparse_template = False
use_opencl_with_sparse = False
if nb_chan > 64 and not peeler_params['use_sparse_template']:
print('OpenCL is not available processing will be slow, try install it')
cc = tdc.CatalogueConstructor(dataio=tdc_dataio, chan_grp=chan_grp)
tdc.apply_all_catalogue_steps(cc, catalogue_nested_params, verbose=self.verbose)
if clean_catalogue_gui:
import pyqtgraph as pg
app = pg.mkQApp()
win = tdc.CatalogueWindow(cc)
win.show()
app.exec_()
if self.verbose:
print(cc)
t0 = time.perf_counter()
cc.make_catalogue_for_peeler()
if self.verbose:
t1 = time.perf_counter()
print('make_catalogue_for_peeler', t1-t0)
# apply Peeler (template matching)
initial_catalogue = tdc_dataio.load_catalogue(chan_grp=chan_grp)
peeler = tdc.Peeler(tdc_dataio)
peeler.change_params(catalogue=initial_catalogue, **peeler_params)
t0 = time.perf_counter()
peeler.run(duration=None, progressbar=False)
if self.verbose:
t1 = time.perf_counter()
print('peeler.tun', t1-t0)
def _run_from_folder(cls, output_folder, params, verbose):
import tridesclous as tdc
tdc_dataio = tdc.DataIO(dirname=str(output_folder))
params = params.copy()
# make catalogue
chan_grps = list(tdc_dataio.channel_groups.keys())
for chan_grp in chan_grps:
# parameters can change depending the group
catalogue_nested_params = make_nested_tdc_params(
tdc_dataio, chan_grp, **params)
if verbose:
print('catalogue_nested_params')
pprint(catalogue_nested_params)
peeler_params = tdc.get_auto_params_for_peelers(
tdc_dataio, chan_grp)
if verbose:
print('peeler_params')
pprint(peeler_params)
cc = tdc.CatalogueConstructor(dataio=tdc_dataio, chan_grp=chan_grp)
tdc.apply_all_catalogue_steps(cc,
catalogue_nested_params,
verbose=verbose)
if verbose:
print(cc)
# apply Peeler (template matching)
initial_catalogue = tdc_dataio.load_catalogue(chan_grp=chan_grp)
peeler = tdc.Peeler(tdc_dataio)
peeler.change_params(catalogue=initial_catalogue, **peeler_params)
t0 = time.perf_counter()
peeler.run(duration=None, progressbar=False)
if verbose:
t1 = time.perf_counter()
print('peeler.tun', t1 - t0)
def _run(self, recording, output_folder):
nb_chan = recording.get_num_channels()
tdc_dataio = tdc.DataIO(dirname=str(output_folder))
# make catalogue
chan_grps = list(tdc_dataio.channel_groups.keys())
for chan_grp in chan_grps:
# parameters can change depending the group
catalogue_nested_params = make_nested_tdc_params(
tdc_dataio, chan_grp, **self.params)
#~ print(catalogue_nested_params)
peeler_params = tdc.get_auto_params_for_peelers(
tdc_dataio, chan_grp)
#~ print(peeler_params)
# check params and OpenCL when many channels
use_sparse_template = False
use_opencl_with_sparse = False
if nb_chan > 64 and not peeler_params['use_sparse_template']:
print(
'OpenCL is not available processing will be slow, try install it'
)
cc = tdc.CatalogueConstructor(dataio=tdc_dataio, chan_grp=chan_grp)
tdc.apply_all_catalogue_steps(
cc,
catalogue_nested_params,
verbose=self.debug,
)
if self.debug:
print(cc)
cc.make_catalogue_for_peeler()
# apply Peeler (template matching)
initial_catalogue = tdc_dataio.load_catalogue(chan_grp=chan_grp)
peeler = tdc.Peeler(tdc_dataio)
peeler.change_params(catalogue=initial_catalogue, **peeler_params)
peeler.run(duration=None, progressbar=self.debug)
def tdc_helper(*, tmpdir, params, recording):
import tridesclous as tdc
# nb_chan = recording.get_num_channels()
# check params and OpenCL when many channels
use_sparse_template = False
use_opencl_with_sparse = False
# if nb_chan > 64: # this limit depend on the platform of course
# if tdc.cltools.HAVE_PYOPENCL:
# # force opencl
# self.params['fullchain_kargs']['preprocessor']['signalpreprocessor_engine'] = 'opencl'
# use_sparse_template = True
# use_opencl_with_sparse = True
# else:
# print('OpenCL is not available processing will be slow, try install it')
tdc_dataio = tdc.DataIO(dirname=str(tmpdir))
# make catalogue
chan_grps = list(tdc_dataio.channel_groups.keys())
for chan_grp in chan_grps:
cc = tdc.CatalogueConstructor(dataio=tdc_dataio, chan_grp=chan_grp)
tdc.apply_all_catalogue_steps(cc, verbose=True, **params)
cc.make_catalogue_for_peeler()
# apply Peeler (template matching)
initial_catalogue = tdc_dataio.load_catalogue(chan_grp=chan_grp)
peeler = tdc.Peeler(tdc_dataio)
peeler.change_params(
catalogue=initial_catalogue,
use_sparse_template=use_sparse_template,
sparse_threshold_mad=1.5,
use_opencl_with_sparse=use_opencl_with_sparse,
)
peeler.run(duration=None, progressbar=False)
sorting = se.TridesclousSortingExtractor(tmpdir)
return sorting
#Compute time vector of the whole trace
sampling_period = 1.0 / sampling_rate
len_trace = 0
for seg_num in range(n_seg):
len_seg = dataio.get_segment_length(seg_num)
len_trace += len_seg
time_vector = np.arange(0, len_trace, 1) * sampling_period
#Mean episode lenght
ep_len = (len_trace / n_seg) * sampling_period
for chan_grp in channel_groups:
#Define the constructor and the channel group
cc = tdc.CatalogueConstructor(dataio, chan_grp=chan_grp)
print('--- Experiment : {} ---'.format(name))
print('Catalogue loaded from {}'.format(path))
print('----Channel group {}----'.format(chan_grp))
#The cluster list for the spikes
clust_id = cc.all_peaks['cluster_label']
#The spike times
spike_index = cc.all_peaks['index']
spike_times = spike_index * sampling_period
seg_id = cc.all_peaks['segment'] #The segment list for the spikes
for ind in range(len(spike_times)):
spike_times[ind] = spike_times[ind] + (ep_len * seg_id[ind])
#Stim vector for the whole trace
def main():
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description='tridesclous')
parser.add_argument('command',
help='command in [{}]'.format(txt_command_list),
default='mainwin',
nargs='?')
parser.add_argument('-d',
'--dirname',
help='working directory',
default=None)
parser.add_argument('-c',
'--chan_grp',
type=int,
help='channel group index',
default=0)
parser.add_argument('-p',
'--parameters',
help='JSON parameter file',
default=None)
args = parser.parse_args(argv)
#~ print(sys.argv)
#~ print(args)
#~ print(args.command)
command = args.command
if not command in comand_list:
print('command should be in [{}]'.format(txt_command_list))
exit()
dirname = args.dirname
if dirname is None:
dirname = os.getcwd()
#~ print(command)
if command in ['cataloguewin', 'peelerwin']:
if not tdc.DataIO.check_initialized(dirname):
print('{} is not initialized'.format(dirname))
exit()
dataio = tdc.DataIO(dirname=dirname)
print(dataio)
if command == 'mainwin':
open_mainwindow()
elif command == 'makecatalogue':
pass
elif command == 'runpeeler':
pass
elif command == 'cataloguewin':
catalogueconstructor = tdc.CatalogueConstructor(dataio=dataio,
chan_grp=args.chan_grp)
app = pg.mkQApp()
win = tdc.CatalogueWindow(catalogueconstructor)
win.show()
app.exec_()
elif command == 'peelerwin':
initial_catalogue = dataio.load_catalogue(chan_grp=args.chan_grp)
app = pg.mkQApp()
win = tdc.PeelerWindow(dataio=dataio, catalogue=initial_catalogue)
win.show()
app.exec_()
elif command == 'init':
app = pg.mkQApp()
win = tdc.InitializeDatasetWindow()
win.show()
app.exec_()
from sklearn.metrics import silhouette_samples, silhouette_score
from sklearn.cluster import KMeans, MeanShift
from sklearn.mixture import GaussianMixture
#~ p = '../example/'
#~ dirname = p + 'tridesclous_locust'
#~ dirname = p +'tridesclous_olfactory_bulb'
#~ dirname = p +'tridesclous_purkinje'
#~ dirname = '/media/samuel/SamCNRS/DataSpikeSorting/pierre/GT 252/tridesclous_GT256_20160426'
#~ dirname = '/home/samuel/Documents/projet/DataSpikeSorting/david robbe/test_2017_03_24_14_35/tdc_test_2017_03_24_14_35/'
dirname = '/home/samuel/Documents/projet/DataSpikeSorting/kampff/2014_11_25_Pair_3_0/tdc_adc2014-11-25T23_00_08/'
dataio = tdc.DataIO(dirname=dirname)
cc = catalogueconstructor = tdc.CatalogueConstructor(dataio=dataio)
print(cc)
#~ wf = cc.some_waveforms
#~ data = wf.reshape(wf.shape[0], -1)
def test_silouhette_or_bic():
data = cc.some_features
labels = cc.all_peaks['label'][cc.some_peaks_index]
keep = labels >= 0
labels = labels[keep]
data = data[keep]