def change_channel_color(self, channelidx, color):
self.model.change_channel_color(self.model.get_channel(channelidx),
color)
# Signal.
log.debug("Changed color of channel {0:d} to {1:d}.".format(
channelidx, color))
self.channelColorChanged.emit(channelidx, color)
def open_spikes(self):
"""Open a HDF5 kwik file."""
if not os.path.exists(self.filename_kwik) and os.path.exists(self.filenames['fet']):
klusters_to_hdf5(self.filename, self.klusters_to_hdf5_progress_report)
self.initialize_logfile()
# Load the similarity measure chosen by the user in the preferences
# file: 'gaussian' or 'kl'.
# Refresh the preferences file when a new file is opened.
# USERPREF.refresh()
self.similarity_measure = self.userpref['similarity_measure'] or 'gaussian'
debug("Similarity measure: {0:s}.".format(self.similarity_measure))
info("Opening {0:s}.".format(self.filename))
if os.path.exists(self.filename):
self.kwik = tb.openFile(self.filename, mode='r+')
self.read_metadata(self.kwik)
# Get the list of shanks.
# WARNING
# The commented code below detects the shank indices from introspection
# in the "shanks" group. It is not necessary anymore as soon as the
# metadata contains a "SHANKS" attribute with the list of shanks.
self.shanks = [int(re.match("shank([0-9]+)",
shank._v_name).group(1))
for shank in self.kwik.listNodes('/shanks')]
print self.shanks
# By default, read the first available shank.
self.set_shank(self.shanks[0])
self.read_shank()
def _correlograms_computed(self, clusters, correlograms, ncorrbins, corrbin, sample_rate, wizard):
clusters_selected = self.loader.get_clusters_selected()
# Abort if the selection has changed during the computation of the
# correlograms.
# Reset the cursor.
self.mainwindow.set_busy(computing_correlograms=False)
if not np.array_equal(clusters, clusters_selected):
log.debug(
"Skip update correlograms with clusters selected={0:s}"
" and clusters updated={1:s}.".format(clusters_selected, clusters)
)
return
if self.statscache.ncorrbins != ncorrbins:
log.debug(
(
"Skip updating correlograms because ncorrbins has "
"changed (from {0:d} to {1:d})".format(ncorrbins, self.statscache.ncorrbins)
)
)
return
# Put the computed correlograms in the cache.
self.statscache.correlograms.update(clusters, correlograms)
# Update the view.
# self.update_correlograms_view()
return ("_update_correlograms_view", (), dict(wizard=wizard))
def add_ipython_view(self, floating=None):
view = self.create_view(vw.IPythonView,
index=len(self.views['IPythonView']),
position=QtCore.Qt.BottomDockWidgetArea,
floating=True)
# Create namespace for the interactive session.
namespace = dict(
window=self,
select=self.get_view('ClusterView').select,
loader=self.loader,
stats=self.statscache,
wizard=self.wizard,
)
view.set_data(**namespace)
# Load all .py files in the code directory.
paths = USERPREF['ipython_import_paths'] or []
if isinstance(paths, basestring):
paths = [paths]
for path in paths:
path = os.path.realpath(os.path.expanduser(path))
if os.path.exists(path):
files = [file for file in os.listdir(path) if file.endswith('.py')]
for file in files:
log.debug("Running {0:s}".format(file))
view.run_file(os.path.join(path, file))
self.views['IPythonView'].append(view)
def find_candidates(self, target):
if target is None:
return []
# Relative target.
try:
target_rel = np.nonzero(self.clusters_unique == target)[0][0]
except IndexError:
log.debug("Target cluster {0:d} does not exist.".format(target))
return []
hidden = self.cluster_groups <= 1
# Hide values in the matrix for hidden clusters.
matrix = self.matrix.copy()
matrix[hidden, :] = -1
matrix[:, hidden] = -1
n = matrix.shape[0]
# Sort all neighbor clusters.
clusters_rel = np.argsort(
np.hstack((matrix[target_rel, :],
matrix[:, target_rel])))[::-1] % n
# Remove duplicates and preserve the order.
clusters_rel = unique(clusters_rel)
clusters_rel.remove(target_rel)
# Remove hidden clusters.
[clusters_rel.remove(cl) for cl in np.nonzero(hidden)[0]
if cl in clusters_rel]
candidates = self.clusters_unique[clusters_rel]
return candidates
def change_cluster_color(self, clusteridx, color):
self.model.change_cluster_color(self.model.get_cluster(clusteridx),
color)
# Signal.
log.debug("Changed color of cluster {0:d} to {1:d}.".format(
clusteridx, color))
self.clusterColorChanged.emit(clusteridx, color)
def select_pair(self, parameter, add=False):
if self.data_manager.nclusters_displayed == 0:
return
nav = self.get_processor('navigation')
# window coordinates
x, y = parameter
# data coordinates
xd, yd = nav.get_data_coordinates(x, y)
cx_rel, cy_rel = self.info_manager.get_closest_cluster(xd, yd)
cx = self.data_manager.clusters_displayed[cx_rel]
cy = self.data_manager.clusters_displayed[cy_rel]
if cx != cy:
clusters = np.array([cx, cy])
else:
clusters = np.array([cx])
if add:
clusters = np.array(sorted(set(self.clusters_selected).union(
clusters)))
# clusters_new = clusters
# clusters = self.clusters_selected.extend([cluster
# for cluster in clusters_new
# if cluster not in self.clusters_selected])
self.clusters_selected = clusters
# Emit signal.
log.debug("Selected clusters {0:s}.".format(str(clusters)))
self.parent.clustersSelected.emit(clusters)
def _compute_similarity_matrix(self, target_next=None):
similarity_measure = self.loader.similarity_measure
features = self.loader.background_features
masks = self.loader.background_masks
clusters = get_array(self.loader.get_clusters(
spikes=self.loader.background_spikes))
cluster_groups = get_array(self.loader.get_cluster_groups('all'))
clusters_all = self.loader.get_clusters_unique()
# Get cluster indices that need to be updated.
# if clusters_to_update is None:
# NOTE: not specifying explicitely clusters_to_update ensures that
# all clusters that need to be updated are updated.
# Allows to fix a bug where the matrix is not updated correctly
# when multiple calls to this functions are called quickly.
clusters_to_update = (self.statscache.similarity_matrix.
not_in_key_indices(clusters_all))
log.debug("Clusters to update: {0:s}".format(str(clusters_to_update)))
# If there are pairs that need to be updated, launch the task.
if len(clusters_to_update) > 0:
self.mainwindow.set_busy(computing_matrix=True)
# Launch the task.
self.tasks.similarity_matrix_task.compute(features,
clusters, cluster_groups, masks, clusters_to_update,
target_next=target_next, similarity_measure=similarity_measure)
# Otherwise, update directly the correlograms view without launching
# the task in the external process.
else:
return [('_wizard_update', (target_next,)),
('_update_similarity_matrix_view',),
]
def _compute_similarity_matrix(self, target_next=None):
exp = self.experiment
channel_group = self.loader.shank
clustering = "main" # TODO
fetdim = exp.application_data.spikedetekt.n_features_per_channel
clusters_data = getattr(exp.channel_groups[channel_group].clusters, clustering)
spikes_data = exp.channel_groups[channel_group].spikes
cluster_groups_data = getattr(exp.channel_groups[channel_group].cluster_groups, clustering)
clusters_all = sorted(clusters_data.keys())
cluster_groups = pd.Series([clusters_data[cl].cluster_group or 0 for cl in clusters_all], index=clusters_all)
spikes_selected, fm = spikes_data.load_features_masks(fraction=0.1)
clusters = getattr(spikes_data.clusters, clustering)[:][spikes_selected]
fm = np.atleast_3d(fm)
features = fm[:, :, 0]
if features.shape[1] <= 1:
return []
# masks = fm[:, ::fetdim, 1]
if fm.shape[2] > 1:
masks = fm[:, :, 1]
else:
masks = None
# features = pandaize(features, spikes_selected)
# masks = pandaize(masks, spikes_selected)
# Get cluster indices that need to be updated.
# if clusters_to_update is None:
# NOTE: not specifying explicitely clusters_to_update ensures that
# all clusters that need to be updated are updated.
# Allows to fix a bug where the matrix is not updated correctly
# when multiple calls to this functions are called quickly.
clusters_to_update = self.statscache.similarity_matrix.not_in_key_indices(clusters_all)
log.debug("Clusters to update: {0:s}".format(str(clusters_to_update)))
# If there are pairs that need to be updated, launch the task.
if len(clusters_to_update) > 0:
self.mainwindow.set_busy(computing_matrix=True)
# Launch the task.
self.tasks.similarity_matrix_task.compute(
features,
clusters,
cluster_groups,
masks,
clusters_to_update,
target_next=target_next,
similarity_measure=None,
)
# Otherwise, update directly the correlograms view without launching
# the task in the external process.
else:
return [("_wizard_update", (target_next,)), ("_update_similarity_matrix_view",)]
def move_channels(self, channels, groupidx):
if not hasattr(channels, '__len__'):
channels = [channels]
if len(channels) == 0:
return
# Signal.
log.debug("Moving channels {0:s} to group {1:d}.".format(
str(channels), groupidx))
self.channelsMoved.emit(np.array(channels), groupidx)
def remove_empty_clusters(self):
clusters_all = self.cluster_groups.index
clusters_in_data = self.clusters_unique
clusters_empty = sorted(set(clusters_all) - set(clusters_in_data))
if len(clusters_empty) > 0:
debug("Removing empty clusters {0:s}.".format(str(clusters_empty)))
for cluster in clusters_empty:
self.remove_cluster(cluster)
return clusters_empty
def move_clusters(self, clusters, groupidx):
if not hasattr(clusters, '__len__'):
clusters = [clusters]
if len(clusters) == 0:
return
# Signal.
log.debug("Moving clusters {0:s} to group {1:d}.".format(
str(clusters), groupidx))
self.clustersMoved.emit(np.array(clusters), groupidx)
def set_projection(self, coord, channel, feature, do_emit=True):
if feature == -1:
feature = self.projection_manager.get_smart_feature(coord, channel)
log.debug(("Set projection on channel {0:d}, feature {1:d} "
"on coord {2:s}".format(channel, feature, 'xy'[coord])))
self.projection_manager.set_projection(coord, channel, feature)
if do_emit:
self.projectionChanged.emit(coord, channel, feature)
self.paint_manager.update_points()
self.paint_manager.updateGL()
def compute(self, features, clusters,
cluster_groups, masks, clusters_selected, target_next=None,
similarity_measure=None):
log.debug("Computing correlation for clusters {0:s}.".format(
str(list(clusters_selected))))
if len(clusters_selected) == 0:
return {}
correlations = compute_correlations(features, clusters,
masks, clusters_selected, similarity_measure=similarity_measure)
return correlations
def compute(self, features, clusters,
cluster_groups, masks, clusters_selected, target_next=None,
similarity_measure=None):
log.debug("Computing correlation for clusters {0:s}.".format(
str(list(clusters_selected))))
if len(clusters_selected) == 0:
return {}
correlations = compute_correlations(features, clusters,
masks, clusters_selected, similarity_measure=similarity_measure)
return correlations
def compute(self, spiketimes, clusters, clusters_to_update=None,
clusters_selected=None, ncorrbins=None, corrbin=None, wizard=None):
log.debug("Computing correlograms for clusters {0:s}.".format(
str(list(clusters_to_update))))
if len(clusters_to_update) == 0:
return {}
clusters_to_update = np.array(clusters_to_update, dtype=np.int32)
correlograms = compute_correlograms(spiketimes, clusters,
clusters_to_update=clusters_to_update,
ncorrbins=ncorrbins, corrbin=corrbin)
return correlograms
def compute(self, features, clusters,
cluster_groups, masks, clusters_selected, target_next=None,
similarity_measure=None):
log.debug("Computing correlation for clusters {0:s}.".format(
str(list(clusters_selected))))
if len(clusters_selected) == 0:
return {}
if self.sm is None:
self.sm = SimilarityMatrix(features, masks)
correlations = self.sm.compute_matrix(clusters, clusters_selected)
return correlations
def compute(self, features, clusters,
cluster_groups, masks, clusters_selected, target_next=None,
similarity_measure=None):
log.debug("Computing correlation for clusters {0:s}.".format(
str(list(clusters_selected))))
if len(clusters_selected) == 0:
return {}
if self.sm is None:
self.sm = SimilarityMatrix(features, masks)
correlations = self.sm.compute_matrix(clusters, clusters_selected)
return correlations
def compute(self, spiketimes, clusters, clusters_to_update=None,
clusters_selected=None, ncorrbins=None, corrbin=None, wizard=None):
log.debug("Computing correlograms for clusters {0:s}.".format(
str(list(clusters_to_update))))
if len(clusters_to_update) == 0:
return {}
clusters_to_update = np.array(clusters_to_update, dtype=np.int32)
correlograms = compute_correlograms(spiketimes, clusters,
clusters_to_update=clusters_to_update,
ncorrbins=ncorrbins, corrbin=corrbin)
return correlograms
def add_group(self, name, channels=[]):
color = random_color()
group = self.model.add_group(name, color)
groupidx = group.groupidx()
# Signal.
log.debug("Adding group {0:s}.".format(name))
self.groupAdded.emit(groupidx, name, color)
# Move the selected channels to the new group.
if channels:
self.move_channels(channels, groupidx)
self.expandAll()
return groupidx
def open(self, filename=None):
"""Open everything."""
if filename is None:
filename = self.filename
else:
self.filename = filename
dir, basename = os.path.split(filename)
# Converting to kwik if needed
# kwik = find_filename(basename, 'kwik', dir=dir)
# xml = find_filename(basename, 'xml', dir=dir)
# self.filename_clu = find_filename(basename, 'clu', dir=dir)
self._filenames = find_filenames(filename)
kwik = find_filename(basename, 'kwik', dir=dir)
xml = self._filenames['xml']
clu = self._filenames['clu']
self.log_filename = find_filename_or_new(filename, 'kvlog', dir=dir)
# Backup the .clu file.
clu_original = find_filename_or_new(filename, 'clu_original')
if os.path.exists(clu) and not os.path.exists(clu_original):
shutil.copyfile(clu, clu_original)
if not kwik:
assert xml, ValueError("I need the .xml file!")
klusters_to_kwik(filename=xml, dir=dir,
progress_report=self._report_progress_open)
self.experiment = Experiment(basename, dir=dir, mode='a')
# CONSISTENCY CHECK
# add missing clusters
add_missing_clusters(self.experiment)
# TODO
# self.initialize_logfile()
# Load the similarity measure chosen by the user in the preferences
# file: 'gaussian' or 'kl'.
# Refresh the preferences file when a new file is opened.
# USERPREF.refresh()
self.similarity_measure = self.userpref['similarity_measure'] or 'gaussian'
debug("Similarity measure: {0:s}.".format(self.similarity_measure))
info("Opening {0:s}.".format(self.experiment.name))
self.shanks = sorted(self.experiment.channel_groups.keys())
self.freq = self.experiment.application_data.spikedetekt.sample_rate
self.fetdim = self.experiment.application_data.spikedetekt.nfeatures_per_channel
self.nsamples = self.experiment.application_data.spikedetekt.waveforms_nsamples
self.set_shank(self.shanks[0])
def add_group(self, name, channels=[]):
color = random_color()
group = self.model.add_group(name, color)
groupidx = group.groupidx()
# Signal.
log.debug("Adding group {0:s}.".format(name))
self.groupAdded.emit(groupidx, name, color)
# Move the selected channels to the new group.
if channels:
self.move_channels(channels, groupidx)
self.expandAll()
return groupidx
def open(self, filename=None):
"""Open everything."""
if filename is None:
filename = self.filename
else:
self.filename = filename
dir, basename = os.path.split(filename)
# Converting to kwik if needed
# kwik = find_filename(basename, 'kwik', dir=dir)
# xml = find_filename(basename, 'xml', dir=dir)
# self.filename_clu = find_filename(basename, 'clu', dir=dir)
self._filenames = find_filenames(filename)
kwik = find_filename(basename, 'kwik', dir=dir)
xml = self._filenames['xml']
clu = self._filenames['clu']
self.log_filename = find_filename_or_new(filename, 'kvlog', dir=dir)
# Backup the .clu file.
clu_original = find_filename_or_new(filename, 'clu_original')
if os.path.exists(clu) and not os.path.exists(clu_original):
shutil.copyfile(clu, clu_original)
if not kwik:
assert xml, ValueError("I need a valid .kwik file")
return
self.experiment = Experiment(basename, dir=dir, mode='a')
# CONSISTENCY CHECK
# add missing clusters
add_missing_clusters(self.experiment)
# TODO
# self.initialize_logfile()
# Load the similarity measure chosen by the user in the preferences
# file: 'gaussian' or 'kl'.
# Refresh the preferences file when a new file is opened.
# USERPREF.refresh()
self.similarity_measure = self.userpref[
'similarity_measure'] or 'gaussian'
debug("Similarity measure: {0:s}.".format(self.similarity_measure))
info("Opening {0:s}.".format(self.experiment.name))
self.shanks = sorted(self.experiment.channel_groups.keys())
self.freq = self.experiment.application_data.spikedetekt.sample_rate
self.fetdim = self.experiment.application_data.spikedetekt.n_features_per_channel
self.nsamples = self.experiment.application_data.spikedetekt.extract_s_before + self.experiment.application_data.spikedetekt.extract_s_after
self.set_shank(self.shanks[0])
def select_neighbor_channel(self, parameter):
coord, channel_dir = parameter
self.projection_manager.select_neighbor_channel(coord, channel_dir)
channel, feature = self.projection_manager.get_projection(coord)
log.debug(("Projection changed to channel {0:d} and "
"feature {1:d} on axis {2:s}.").format(
channel, feature, 'xy'[coord]))
self.parent.projectionChanged.emit(coord, channel, feature)
self.paint_manager.update_points()
self.paint_manager.updateGL()
def _select_done(self, clusters, wizard=False):
if wizard:
target = (self.wizard.current_target(),)
else:
target = ()
# self.loader.select(clusters=clusters)
log.debug("Selected clusters {0:s}.".format(str(clusters)))
return [
('_update_feature_view', target),
('_update_waveform_view', (), dict(wizard=wizard)),
('_show_selection_in_matrix', (clusters,)),
('_compute_correlograms', (clusters,),),
]
def _select_done(self, clusters, wizard=False,):
if wizard:
target = (self.wizard.current_target(),)
else:
target = ()
# self.loader.select(clusters=clusters)
log.debug("Selected clusters {0:s}.".format(str(clusters)))
return [
('_update_feature_view', target, dict()),
('_update_waveform_view', (), dict(wizard=wizard,)),
('_show_selection_in_matrix', (clusters,),),
('_compute_correlograms', (clusters,), dict(wizard=wizard,)),
]
def test_wizard_merge():
# Create mock data.
clusters = create_clusters(nspikes, nclusters)
cluster_groups = create_cluster_groups(nclusters)
similarity_matrix = create_similarity_matrix(nclusters)
quality = np.diag(similarity_matrix)
# Get the best clusters.
clusters_unique = np.unique(clusters)
target = clusters_unique[np.argmax(quality)]
# Initialize the wizard.
w = Wizard()
w.set_data(similarity_matrix=similarity_matrix,
cluster_groups=cluster_groups)
w.update_candidates()
cluster = w.current_candidate()
# Simulate a merge: target and cluster ==> cluster_new.
cluster_new = clusters_unique.max() + 1
clusters[clusters == target] = cluster_new
clusters[clusters == cluster] = cluster_new
log.debug("Merged {0:d} and {1:d} to {2:d}".format(target, cluster,
cluster_new))
similarity_matrix = create_similarity_matrix(nclusters - 1)
indices = [
x for x in xrange(cluster_offset, cluster_offset + nclusters + 1)
if x != cluster and x != target
]
cluster_groups = pd.Series(np.array(np.ones(nclusters - 1) * 3,
dtype=np.int32),
index=np.array(indices))
# Update the wizard.
quality = np.diag(similarity_matrix)
w.set_data(similarity_matrix=similarity_matrix,
cluster_groups=cluster_groups)
w.update_candidates(cluster_new)
assert w.current_target() == cluster_new
c = w.current_candidate()
assert c is not None
assert w.previous_candidate() == w.current_candidate()
assert w.next_candidate() == c
for _ in xrange(nclusters):
c = w.next_candidate()
assert c not in (target, cluster)
def select_neighbor_channel(self, parameter):
coord, channel_dir = parameter
self.projection_manager.select_neighbor_channel(coord, channel_dir)
channel, feature = self.projection_manager.get_projection(coord)
log.debug(
("Projection changed to channel {0:d} and "
"feature {1:d} on axis {2:s}.").format(channel, feature,
'xy'[coord]))
self.parent.projectionChanged.emit(coord, channel, feature)
self.paint_manager.update_points()
self.paint_manager.updateGL()
def test_wizard_merge():
# Create mock data.
clusters = create_clusters(nspikes, nclusters)
cluster_groups = create_cluster_groups(nclusters)
similarity_matrix = create_similarity_matrix(nclusters)
quality = np.diag(similarity_matrix)
# Get the best clusters.
clusters_unique = np.unique(clusters)
target = clusters_unique[np.argmax(quality)]
# Initialize the wizard.
w = Wizard()
w.set_data(similarity_matrix=similarity_matrix,
cluster_groups=cluster_groups)
w.update_candidates()
cluster = w.current_candidate()
# Simulate a merge: target and cluster ==> cluster_new.
cluster_new = clusters_unique.max() + 1
clusters[clusters == target] = cluster_new
clusters[clusters == cluster] = cluster_new
log.debug("Merged {0:d} and {1:d} to {2:d}".format(
target, cluster, cluster_new))
similarity_matrix = create_similarity_matrix(nclusters - 1)
indices = [x for x in xrange(cluster_offset, cluster_offset + nclusters + 1)
if x != cluster and x != target]
cluster_groups = pd.Series(np.array(np.ones(nclusters - 1) * 3, dtype=np.int32),
index=np.array(indices))
# Update the wizard.
quality = np.diag(similarity_matrix)
w.set_data(similarity_matrix=similarity_matrix,
cluster_groups=cluster_groups)
w.update_candidates(cluster_new)
assert w.current_target() == cluster_new
c = w.current_candidate()
assert c is not None
assert w.previous_candidate() == w.current_candidate()
assert w.next_candidate() == c
for _ in xrange(nclusters):
c = w.next_candidate()
assert c not in (target, cluster)
def select_feature(self, parameter):
coord, feature = parameter
if feature < 0 or feature >= self.data_manager.fetdim:
return
self.projection_manager.select_feature(coord, feature)
channel, feature = self.projection_manager.get_projection(coord)
log.debug(("Projection changed to channel {0:d} and "
"feature {1:d} on axis {2:s}.").format(
channel, feature, 'xy'[coord]))
self.parent.projectionChanged.emit(coord, channel, feature)
self.paint_manager.update_points()
self.paint_manager.updateGL()
def select_feature(self, parameter):
coord, feature = parameter
if feature < 0 or feature >= self.data_manager.fetdim:
return
self.projection_manager.select_feature(coord, feature)
channel, feature = self.projection_manager.get_projection(coord)
log.debug(
("Projection changed to channel {0:d} and "
"feature {1:d} on axis {2:s}.").format(channel, feature,
'xy'[coord]))
self.parent.projectionChanged.emit(coord, channel, feature)
self.paint_manager.update_points()
self.paint_manager.updateGL()
def _read_traces(files, dtype=None, n_channels=None):
kwd_path = None
dat_path = None
kwik = files['kwik']
recordings = kwik.root.recordings
traces = []
# opened_files = []
for recording in recordings:
# Is there a path specified to a .raw.kwd file which exists in
# [KWIK]/recordings/[X]/raw? If so, open it.
raw = recording.raw
if 'hdf5_path' in raw._v_attrs:
kwd_path = raw._v_attrs.hdf5_path[:-8]
kwd = files['raw.kwd']
if kwd is None:
debug("%s not found, trying same basename in KWIK dir" %
kwd_path)
else:
debug("Loading traces: %s" % kwd_path)
traces.append(
kwd.root.recordings._f_getChild(str(
recording._v_name)).data)
# opened_files.append(kwd)
continue
# Is there a path specified to a .dat file which exists?
if 'dat_path' in raw._v_attrs:
dtype = kwik.root.application_data.spikedetekt._v_attrs.dtype[0]
if dtype:
dtype = np.dtype(dtype)
n_channels = kwik.root.application_data.spikedetekt._v_attrs. \
n_channels
if n_channels:
n_channels = int(n_channels)
assert dtype is not None
assert n_channels
dat_path = raw._v_attrs.dat_path
if not op.exists(dat_path):
debug("%s not found, trying same basename in KWIK dir" %
dat_path)
else:
debug("Loading traces: %s" % dat_path)
dat = _dat_to_traces(dat_path,
dtype=dtype,
n_channels=n_channels)
traces.append(dat)
# opened_files.append(dat)
continue
if not traces:
warn("No traces found: the waveforms won't be available.")
return _concatenate_virtual_arrays(traces)
def _correlograms_computed(self, clusters, correlograms, ncorrbins, corrbin, wizard):
clusters_selected = self.loader.get_clusters_selected()
# Abort if the selection has changed during the computation of the
# correlograms.
# Reset the cursor.
self.mainwindow.set_busy(computing_correlograms=False)
if not np.array_equal(clusters, clusters_selected):
log.debug("Skip update correlograms with clusters selected={0:s}"
" and clusters updated={1:s}.".format(clusters_selected, clusters))
return
if self.statscache.ncorrbins != ncorrbins:
log.debug(("Skip updating correlograms because ncorrbins has "
"changed (from {0:d} to {1:d})".format(
ncorrbins, self.statscache.ncorrbins)))
return
# Put the computed correlograms in the cache.
self.statscache.correlograms.update(clusters, correlograms)
# Update the view.
# self.update_correlograms_view()
return ('_update_correlograms_view', (), dict(wizard=wizard))
def _read_traces(files, dtype=None, n_channels=None):
kwd_path = None
dat_path = None
kwik = files['kwik']
recordings = kwik.root.recordings
traces = []
# opened_files = []
for recording in recordings:
# Is there a path specified to a .raw.kwd file which exists in
# [KWIK]/recordings/[X]/raw? If so, open it.
raw = recording.raw
if 'hdf5_path' in raw._v_attrs:
kwd_path = raw._v_attrs.hdf5_path[:-8]
kwd = files['raw.kwd']
if kwd is None:
debug("%s not found, trying same basename in KWIK dir" %
kwd_path)
else:
debug("Loading traces: %s" % kwd_path)
traces.append(kwd.root.recordings._f_getChild(str(recording._v_name)).data)
# opened_files.append(kwd)
continue
# Is there a path specified to a .dat file which exists?
if 'dat_path' in raw._v_attrs:
dtype = kwik.root.application_data.spikedetekt._v_attrs.dtype[0]
if dtype:
dtype = np.dtype(dtype)
n_channels = kwik.root.application_data.spikedetekt._v_attrs. \
n_channels
if n_channels:
n_channels = int(n_channels)
assert dtype is not None
assert n_channels
dat_path = raw._v_attrs.dat_path
if not op.exists(dat_path):
debug("%s not found, trying same basename in KWIK dir" %
dat_path)
else:
debug("Loading traces: %s" % dat_path)
dat = _dat_to_traces(dat_path, dtype=dtype,
n_channels=n_channels)
traces.append(dat)
# opened_files.append(dat)
continue
if not traces:
warn("No traces found: the waveforms won't be available.")
return _concatenate_virtual_arrays(traces)
def select_channel(self, coord, channel):
channel = str(channel).lower()
if channel.startswith('extra'):
channel = channel[6:]
extra = True
else:
extra = False
try:
channel = int(channel)
except ValueError:
log.debug("Unable to parse channel '{0:s}'".format(str(channel)))
channel = self.projection[coord][0]
if extra:
channel += self.nchannels
channel = np.clip(channel, 0, self.nchannels + self.nextrafet - 1)
feature = self.projection[coord][1]
self._change_projection(coord, channel, feature)
def _compute_similarity_matrix(self, target_next=None):
similarity_measure = self.loader.similarity_measure
features = self.loader.background_features
masks = self.loader.background_masks
clusters = get_array(
self.loader.get_clusters(spikes=self.loader.background_spikes))
cluster_groups = get_array(self.loader.get_cluster_groups('all'))
clusters_all = self.loader.get_clusters_unique()
# Get cluster indices that need to be updated.
# if clusters_to_update is None:
# NOTE: not specifying explicitely clusters_to_update ensures that
# all clusters that need to be updated are updated.
# Allows to fix a bug where the matrix is not updated correctly
# when multiple calls to this functions are called quickly.
clusters_to_update = (
self.statscache.similarity_matrix.not_in_key_indices(clusters_all))
log.debug("Clusters to update: {0:s}".format(str(clusters_to_update)))
# If there are pairs that need to be updated, launch the task.
if len(clusters_to_update) > 0:
self.mainwindow.set_busy(computing_matrix=True)
# Launch the task.
self.tasks.similarity_matrix_task.compute(
features,
clusters,
cluster_groups,
masks,
clusters_to_update,
target_next=target_next,
similarity_measure=similarity_measure)
# Otherwise, update directly the correlograms view without launching
# the task in the external process.
else:
return [
('_wizard_update', (target_next, )),
('_update_similarity_matrix_view', ),
]
def read(self):
self.initialize_logfile()
# Load the similarity measure chosen by the user in the preferences
# file: 'gaussian' or 'kl'.
# Refresh the preferences file when a new file is opened.
# USERPREF.refresh()
self.similarity_measure = self.userpref['similarity_measure'] or 'gaussian'
debug("Similarity measure: {0:s}.".format(self.similarity_measure))
info("Opening {0:s}.".format(self.filename))
self.report_progress(0, 5)
self.read_metadata()
self.read_probe()
self.report_progress(1, 5)
self.read_features()
self.report_progress(2, 5)
self.read_res()
self.read_clusters()
self.report_progress(3, 5)
self.read_cluster_info()
self.read_group_info()
self.read_masks()
self.report_progress(4, 5)
self.read_waveforms()
self.report_progress(5, 5)
def find_candidates(self, target):
if target is None:
return []
# Relative target.
try:
target_rel = np.nonzero(self.clusters_unique == target)[0][0]
except IndexError:
log.debug("Target cluster {0:d} does not exist.".format(target))
return []
hidden = self.cluster_groups <= 1
# Hide values in the matrix for hidden clusters.
matrix = self.matrix.copy()
matrix[hidden, :] = -1
matrix[:, hidden] = -1
n = matrix.shape[0]
# Sort all neighbor clusters.
clusters_rel = np.argsort(
np.hstack(
(matrix[target_rel, :], matrix[:, target_rel])))[::-1] % n
# Remove duplicates and preserve the order.
clusters_rel = unique(clusters_rel)
clusters_rel.remove(target_rel)
# Remove hidden clusters.
[
clusters_rel.remove(cl) for cl in np.nonzero(hidden)[0]
if cl in clusters_rel
]
candidates = self.clusters_unique[clusters_rel]
return candidates
def read(self):
self.initialize_logfile()
# Load the similarity measure chosen by the user in the preferences
# file: 'gaussian' or 'kl'.
# Refresh the preferences file when a new file is opened.
# USERPREF.refresh()
self.similarity_measure = self.userpref['similarity_measure'] or 'gaussian'
debug("Similarity measure: {0:s}.".format(self.similarity_measure))
info("Opening {0:s}.".format(self.filename))
self.report_progress(0, 5)
self.read_metadata()
self.read_probe()
self.report_progress(1, 5)
self.read_features()
self.report_progress(2, 5)
self.read_res()
self.read_clusters()
self.report_progress(3, 5)
self.read_cluster_info()
self.read_group_info()
self.read_masks()
self.report_progress(4, 5)
self.read_waveforms()
self.report_progress(5, 5)
def open_preferences_callback(self, checked=None):
url = USERPREF.filepath
log.debug("Opening preferences file at '{0:s}'".format(url))
QtGui.QDesktopServices.openUrl(QtCore.QUrl('file:///' + url))
def __init__(self, parent=None, dolog=True, filename=None):
self.views = {}
super(MainWindow, self).__init__(parent)
self.views = {}
# HACK: display the icon in Windows' taskbar.
if os.name == 'nt':
try:
import ctypes
myappid = 'klustateam.klustaviewa'
ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(myappid)
except:
pass
self.dolog = dolog
if self.dolog:
create_file_logger()
self.initialize_view_logger()
log.debug("Using {0:s}.".format(QT_BINDING))
# Main window options.
self.move(50, 50)
self.setWindowTitle('KlustaViewa')
# Focus options.
self.setFocusPolicy(QtCore.Qt.WheelFocus)
self.setMouseTracking(True)
# Dock widgets options.
self.setDockNestingEnabled(True)
self.setAnimated(False)
self.setWindowIcon(get_icon('logo'))
# Initialize some variables.
self.statscache = None
# self.loader = KlustersLoader()
self.loader = KwikLoader(userpref=USERPREF)
self.loader.progressReported.connect(self.open_progress_reported)
self.loader.saveProgressReported.connect(self.save_progress_reported)
self.wizard = Wizard()
self.controller = None
self.spikes_highlighted = []
self.spikes_selected = []
self._wizard = False
self.is_file_open = False
self.need_save = False
self.taskgraph = TaskGraph(self)
self.busy_cursor = QtGui.QCursor(QtCore.Qt.BusyCursor)
self.normal_cursor = QtGui.QCursor(QtCore.Qt.ArrowCursor)
self.is_busy = False
self.override_color = False
self.computing_correlograms = False
self.computing_matrix = False
# Create the main window.
self.create_views()
self.create_file_actions()
self.create_edit_actions()
self.create_view_actions()
self.create_correlograms_actions()
self.create_control_actions()
self.create_wizard_actions()
self.create_help_actions()
self.create_menu()
self.create_toolbar()
self.create_open_progress_dialog()
self.create_save_progress_dialog()
self.create_threads()
# Update action enabled/disabled property.
self.update_action_enabled()
# Show the main window.
self.set_styles()
self.restore_geometry()
# Automatically load a file upon startup if requested.
if filename:
filename = os.path.realpath(filename)
self.open_task.open(self.loader, filename)
self.show()
def refresh_preferences_callback(self, checked=None):
log.debug("Refreshing user preferences.")
USERPREF.refresh()
def open_preferences_callback(self, checked=None):
url = USERPREF.filepath
log.debug("Opening preferences file at '{0:s}'".format(url))
QtGui.QDesktopServices.openUrl(QtCore.QUrl('file:///' + url))
# -----------------------------------------------------------------------------
# Imports
# -----------------------------------------------------------------------------
import os
from kwiklib.utils import logger as log
try:
from IPython.qt.console.rich_ipython_widget import RichIPythonWidget
from IPython.qt.inprocess import QtInProcessKernelManager
from IPython.lib import guisupport
IPYTHON = True
except Exception as e:
IPYTHON = False
log.debug(("You need IPython 1.0 if you want the IPython console in the"
"application: " + e.message))
import galry
from qtools import QtGui, QtCore
# -----------------------------------------------------------------------------
# IPython view
# -----------------------------------------------------------------------------
class IPythonView(QtGui.QWidget):
def __init__(self, parent=None, getfocus=None):
super(IPythonView, self).__init__(parent)
# Create an in-process kernel
self.kernel_manager = QtInProcessKernelManager()
self.kernel_manager.start_kernel()
def refresh_preferences_callback(self, checked=None):
log.debug("Refreshing user preferences.")
USERPREF.refresh()
def _compute_similarity_matrix(self, target_next=None):
# TODO: get_similarity_matrix_data in viewdata
# return
similarity_measure = self.loader.similarity_measure
# features = self.loader.background_features
# masks = self.loader.background_masks
# clusters = get_array(self.loader.get_clusters(
# spikes=self.loader.background_spikes))
# cluster_groups = get_array(self.loader.get_cluster_groups('all'))
# clusters_all = self.loader.get_clusters_unique()
exp = self.experiment
channel_group = self.loader.shank
clustering = 'main' # TODO
fetdim = exp.application_data.spikedetekt.nfeatures_per_channel
clusters_data = getattr(exp.channel_groups[channel_group].clusters,
clustering)
spikes_data = exp.channel_groups[channel_group].spikes
cluster_groups_data = getattr(
exp.channel_groups[channel_group].cluster_groups, clustering)
clusters_all = sorted(clusters_data.keys())
cluster_groups = pd.Series(
[clusters_data[cl].cluster_group or 0 for cl in clusters_all],
index=clusters_all)
spikes_selected, fm = spikes_data.load_features_masks(fraction=.1)
clusters = getattr(spikes_data.clusters,
clustering)[:][spikes_selected]
fm = np.atleast_3d(fm)
features = fm[:, :, 0]
if features.shape[1] <= 1:
return []
# masks = fm[:, ::fetdim, 1]
if fm.shape[2] > 1:
masks = fm[:, :, 1]
else:
masks = None
# features = pandaize(features, spikes_selected)
# masks = pandaize(masks, spikes_selected)
# Get cluster indices that need to be updated.
# if clusters_to_update is None:
# NOTE: not specifying explicitely clusters_to_update ensures that
# all clusters that need to be updated are updated.
# Allows to fix a bug where the matrix is not updated correctly
# when multiple calls to this functions are called quickly.
clusters_to_update = (
self.statscache.similarity_matrix.not_in_key_indices(clusters_all))
log.debug("Clusters to update: {0:s}".format(str(clusters_to_update)))
# If there are pairs that need to be updated, launch the task.
if len(clusters_to_update) > 0:
self.mainwindow.set_busy(computing_matrix=True)
# Launch the task.
self.tasks.similarity_matrix_task.compute(
features,
clusters,
cluster_groups,
masks,
clusters_to_update,
target_next=target_next,
similarity_measure=similarity_measure)
# Otherwise, update directly the correlograms view without launching
# the task in the external process.
else:
return [
('_wizard_update', (target_next, )),
('_update_similarity_matrix_view', ),
]
def change_group_color(self, groupidx, color):
self.model.change_group_color(self.model.get_group(groupidx), color)
# Signal.
log.debug("Changed color of group {0:d} to {1:d}.".format(
groupidx, color))
self.groupColorChanged.emit(groupidx, color)
def remove_group(self, groupidx):
self.model.remove_group(self.model.get_group(groupidx))
# Signal.
log.debug("Removed group {0:d}.".format(groupidx))
self.groupRemoved.emit(groupidx)
def __init__(self, files, node=None, root=None):
super(Spikes, self).__init__(files, node, root=root)
self.time_samples = self._node.time_samples
self.time_fractional = self._node.time_fractional
self.recording = self._node.recording
self.clusters = Clusters(self._files, self._node.clusters, root=self._root)
# Add concatenated time samples
self.concatenated_time_samples = self._compute_concatenated_time_samples()
self.channel_group_id = self._node._v_parent._v_name
# Get large datasets, that may be in external files.
# self.features_masks = self._get_child('features_masks')
# self.waveforms_raw = self._get_child('waveforms_raw')
# self.waveforms_filtered = self._get_child('waveforms_filtered')
# Load features masks directly from KWX.
g = self.channel_group_id
path = '/channel_groups/{}/features_masks'.format(g)
if files['kwx']:
self.features_masks = files['kwx'].getNode(path)
else:
self.features_masks = None
# Load raw data directly from raw data.
traces = _read_traces(files)
b = self._root.application_data.spikedetekt._f_getAttr('extract_s_before')
a = self._root.application_data.spikedetekt._f_getAttr('extract_s_after')
order = self._root.application_data.spikedetekt._f_getAttr('filter_butter_order')
rate = self._root.application_data.spikedetekt._f_getAttr('sample_rate')
low = self._root.application_data.spikedetekt._f_getAttr('filter_low')
if 'filter_high_factor' in self._root.application_data.spikedetekt._v_attrs:
high = self._root.application_data.spikedetekt._f_getAttr('filter_high_factor') * rate
else:
# NOTE: old format
high = self._root.application_data.spikedetekt._f_getAttr('filter_high')
b_filter = bandpass_filter(rate=rate,
low=low,
high=high,
order=order)
debug("Enable waveform filter.")
def the_filter(x, axis=0):
return apply_filter(x, b_filter, axis=axis)
filter_margin = order * 3
channels = self._root.channel_groups._f_getChild(self.channel_group_id)._f_getAttr('channel_order')
_waveform_loader = WaveformLoader(n_samples=(b, a),
traces=traces,
filter=the_filter,
filter_margin=filter_margin,
scale_factor=.01,
channels=channels,
)
self.waveforms_raw = SpikeLoader(_waveform_loader,
self.concatenated_time_samples)
self.waveforms_filtered = self.waveforms_raw
nspikes = len(self.time_samples)
if self.waveforms_raw is not None:
self.nsamples, self.nchannels = self.waveforms_raw.shape[1:]
if self.features_masks is None:
self.features_masks = np.zeros((nspikes, 1, 1), dtype=np.float32)
if len(self.features_masks.shape) == 3:
self.features = ArrayProxy(self.features_masks, col=0)
self.masks = ArrayProxy(self.features_masks, col=1)
elif len(self.features_masks.shape) == 2:
self.features = self.features_masks
self.masks = None #np.ones_like(self.features)
self.nfeatures = self.features.shape[1]
"""This module implements the computation of the cross-correlograms between
clusters."""
# -----------------------------------------------------------------------------
# Imports
# -----------------------------------------------------------------------------
from itertools import product
import numpy as np
from kwiklib.utils import logger as log
# Trying to load the Cython version.
try:
from correlograms_cython import compute_correlograms_cython as compute_correlograms
log.debug(("Trying to load the compiled Cython version of the correlograms"
"computations..."))
except Exception as e:
log.debug(e.message)
try:
log.debug(("failed. Trying to use Cython directly..."))
import pyximport
pyximport.install(setup_args={'include_dirs': np.get_include()})
from correlograms_cython import compute_correlograms_cython as compute_correlograms
except Exception as e:
log.debug(e.message)
log.info(("Unable to load the fast Cython version of the correlograms"
"computations, so falling back to the pure Python version."))
# Pure Python version.
# --------------------
def compute_correlograms(spiketimes,
# -----------------------------------------------------------------------------
# Imports
# -----------------------------------------------------------------------------
import os
import kwiklib.utils.logger as log
try:
from IPython.qt.console.rich_ipython_widget import RichIPythonWidget
from IPython.qt.inprocess import QtInProcessKernelManager
from IPython.lib import guisupport
IPYTHON = True
except Exception as e:
IPYTHON = False
log.debug(("You need IPython 1.0 if you want the IPython console in the"
"application: " + e.message))
import galry
from qtools import QtGui, QtCore
# -----------------------------------------------------------------------------
# IPython view
# -----------------------------------------------------------------------------
class IPythonView(QtGui.QWidget):
def __init__(self, parent=None, getfocus=None):
super(IPythonView, self).__init__(parent)
# Create an in-process kernel
self.kernel_manager = QtInProcessKernelManager()
self.kernel_manager.start_kernel()
def __init__(self, files, node=None, root=None):
super(Spikes, self).__init__(files, node, root=root)
self.time_samples = self._node.time_samples
self.time_fractional = self._node.time_fractional
self.recording = self._node.recording
self.clusters = Clusters(self._files,
self._node.clusters,
root=self._root)
# Add concatenated time samples
self.concatenated_time_samples = self._compute_concatenated_time_samples(
)
self.channel_group_id = self._node._v_parent._v_name
# Get large datasets, that may be in external files.
# self.features_masks = self._get_child('features_masks')
# self.waveforms_raw = self._get_child('waveforms_raw')
# self.waveforms_filtered = self._get_child('waveforms_filtered')
# Load features masks directly from KWX.
g = self.channel_group_id
path = '/channel_groups/{}/features_masks'.format(g)
if files['kwx']:
self.features_masks = files['kwx'].getNode(path)
else:
self.features_masks = None
# Load raw data directly from raw data.
traces = _read_traces(files)
b = self._root.application_data.spikedetekt._f_getAttr(
'extract_s_before')
a = self._root.application_data.spikedetekt._f_getAttr(
'extract_s_after')
order = self._root.application_data.spikedetekt._f_getAttr(
'filter_butter_order')
rate = self._root.application_data.spikedetekt._f_getAttr(
'sample_rate')
low = self._root.application_data.spikedetekt._f_getAttr('filter_low')
if 'filter_high_factor' in self._root.application_data.spikedetekt._v_attrs:
high = self._root.application_data.spikedetekt._f_getAttr(
'filter_high_factor') * rate
else:
# NOTE: old format
high = self._root.application_data.spikedetekt._f_getAttr(
'filter_high')
b_filter = bandpass_filter(rate=rate, low=low, high=high, order=order)
debug("Enable waveform filter.")
def the_filter(x, axis=0):
return apply_filter(x, b_filter, axis=axis)
filter_margin = order * 3
channels = self._root.channel_groups._f_getChild(
self.channel_group_id)._f_getAttr('channel_order')
_waveform_loader = WaveformLoader(
n_samples=(b + a),
traces=traces,
filter=the_filter,
filter_margin=filter_margin,
scale_factor=.01,
channels=channels,
)
self.waveforms_raw = SpikeLoader(_waveform_loader,
self.time_samples[:])
self.waveforms_filtered = self.waveforms_raw
nspikes = len(self.time_samples)
if self.waveforms_raw is not None:
self.nsamples, self.nchannels = self.waveforms_raw.shape[1:]
if self.features_masks is None:
self.features_masks = np.zeros((nspikes, 1, 1), dtype=np.float32)
if len(self.features_masks.shape) == 3:
self.features = ArrayProxy(self.features_masks, col=0)
self.masks = ArrayProxy(self.features_masks, col=1)
elif len(self.features_masks.shape) == 2:
self.features = self.features_masks
self.masks = None #np.ones_like(self.features)
self.nfeatures = self.features.shape[1]
def rename_channel_group(self, groupidx, name):
self.model.rename_channel_group(self.model.get_group(groupidx), name)
# Signal.
log.debug("Rename group {0:d} to {1:s}.".format(groupidx, name))
self.groupRenamed.emit(groupidx, name)
def rename_channel(self, channelidx, name):
self.model.rename_channel(self.model.get_channel(channelidx), name)
# Signal.
log.debug("Rename channel {0:d} to {1:s}.".format(channelidx, name))
self.channelRenamed.emit(channelidx, name)
def __init__(self, parent=None, dolog=True, filename=None):
super(KwikSkope, self).__init__(parent)
# HACK: display the icon in Windows' taskbar.
if os.name == 'nt':
try:
import ctypes
myappid = 'klustateam.kwikskope'
ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(
myappid)
except:
pass
self.dolog = dolog
if self.dolog:
create_file_logger()
log.debug("Using {0:s}.".format(QT_BINDING))
# Main window options.
self.move(50, 50)
self.setWindowTitle('KwikSkope')
# Focus options.
self.setFocusPolicy(QtCore.Qt.WheelFocus)
self.setMouseTracking(True)
# Dock widgets options.
self.setDockNestingEnabled(True)
self.setAnimated(False)
self.setWindowIcon(get_icon('logo'))
# Initialize some variables.
# self.statscache = None
# self.loader = KlustersLoader()
self.loader = HDF5Loader()
self.loader.progressReported.connect(self.open_progress_reported)
self.loader.saveProgressReported.connect(self.save_progress_reported)
self.wizard = Wizard()
self.controller = None
self.spikes_highlighted = []
self.spikes_selected = []
self._wizard = False
self.is_file_open = False
self.need_save = False
self.busy_cursor = QtGui.QCursor(QtCore.Qt.BusyCursor)
self.normal_cursor = QtGui.QCursor(QtCore.Qt.ArrowCursor)
self.is_busy = False
self.override_color = False
self.computing_correlograms = False
self.computing_matrix = False
# Create the main window.
self.create_views()
self.create_file_actions()
self.create_edit_actions()
self.create_view_actions()
self.create_help_actions()
self.create_menu()
self.create_toolbar()
self.create_open_progress_dialog()
self.create_save_progress_dialog()
self.create_threads()
# Update action enabled/disabled property.
self.update_action_enabled()
# Show the main window.
self.set_styles()
self.restore_geometry()
# Automatically load a file upon startup if requested.
if filename:
filename = os.path.realpath(filename)
self.open_task.open(self.loader, filename)
self.show()
