def renumber_clusters(clusters, cluster_info):
clusters_unique = get_array(get_indices(cluster_info))
nclusters = len(clusters_unique)
assert np.array_equal(clusters_unique, np.unique(clusters))
clusters_array = get_array(clusters)
groups = get_array(cluster_info['group'])
colors = get_array(cluster_info['color'])
groups_unique = np.unique(groups)
# Reorder clusters according to the group.
clusters_unique_reordered = np.hstack(
[sorted(clusters_unique[groups == group]) for group in groups_unique])
# WARNING: there's a +2 offset to avoid conflicts with the old convention
# cluster 0 = noise, cluster 1 = MUA.
clusters_renumbered = reorder(clusters_array,
clusters_unique_reordered) + 2
cluster_permutation = reorder(clusters_unique_reordered, clusters_unique)
# Reorder cluster info.
groups_reordered = groups[cluster_permutation]
colors_reordered = colors[cluster_permutation]
# Recreate new cluster info.
cluster_info_reordered = pd.DataFrame(
{
'color': colors_reordered,
'group': groups_reordered
},
dtype=np.int32,
index=(np.arange(nclusters) + 2))
return clusters_renumbered, cluster_info_reordered
def get_channel_colors(self, channels=None, can_override=True):
if channels is None:
channels = self.channels_selected
if can_override and self.override_color:
channel_group_colors = get_array(self.get_channel_group_colors("all"))
channel_groups = get_array(self.get_channel_groups("all"))
colors = pd.Series(channel_group_colors[channel_groups], index=self.channels)
else:
colors = self.channel_colors
return select(colors, channels)
def get_cluster_colors(self, clusters=None, can_override=True):
if clusters is None:
clusters = self.clusters_selected
if can_override and self.override_color:
group_colors = get_array(self.get_group_colors("all"))
groups = get_array(self.get_cluster_groups("all"))
colors = pd.Series(group_colors[groups], index=self.get_clusters_unique())
else:
colors = pd.Series([self.get_cluster_color(c) for c in clusters], index=clusters)
return select(colors, clusters)
def get_next_cluster(self, cluster):
cluster_groups = self.get_cluster_groups("all")
group = get_array(self.get_cluster_groups(cluster))
clusters = get_indices(cluster_groups)
cluster_groups = get_array(cluster_groups)
samegroup = (cluster_groups == group) & (clusters > cluster)
i = np.nonzero(samegroup)[0]
if len(i) > 0:
return clusters[i[0]]
else:
return cluster
def get_next_cluster(self, cluster):
cluster_groups = self.get_cluster_groups('all')
group = get_array(self.get_cluster_groups(cluster))
clusters = get_indices(cluster_groups)
cluster_groups = get_array(cluster_groups)
samegroup = (cluster_groups == group) & (clusters > cluster)
i = np.nonzero(samegroup)[0]
if len(i) > 0:
return clusters[i[0]]
else:
return cluster
def get_cluster_colors(self, clusters=None, can_override=True,
):
if clusters is None:
clusters = self.clusters_selected
if can_override and self.override_color:
group_colors = get_array(self.get_group_colors('all'))
groups = get_array(self.get_cluster_groups('all'))
colors = pd.Series(group_colors[groups],
index=self.get_clusters_unique())
else:
colors = self.cluster_colors
return select(colors, clusters)
def get_channel_colors(self, channels=None, can_override=True,
):
if channels is None:
channels = self.channels_selected
if can_override and self.override_color:
channel_group_colors = get_array(self.get_channel_group_colors('all'))
channel_groups = get_array(self.get_channel_groups('all'))
colors = pd.Series(channel_group_colors[channel_groups],
index=self.channels)
else:
colors = self.channel_colors
return select(colors, channels)
def _update_data(self,):
"""Update internal variables."""
clusters_array = get_array(self.clusters)
self.clusters_unique = np.unique(clusters_array)
self.nclusters = len(self.clusters_unique)
bincount = np.bincount(clusters_array)
self.counter = {key: bincount[key] for key in np.nonzero(bincount)[0]}
def _update_data(self,):
"""Update internal variables."""
clusters_array = get_array(self.clusters)
self.clusters_unique = np.unique(clusters_array)
self.nclusters = len(self.clusters_unique)
bincount = np.bincount(clusters_array)
self.counter = {key: bincount[key] for key in np.nonzero(bincount)[0]}
def save(self, renumber=False):
self.update_cluster_info()
self.update_group_info()
if renumber:
self.renumber()
clusters = get_array(self.clusters_renumbered)
cluster_info = self.cluster_info_renumbered
else:
clusters = get_array(self.clusters)
cluster_info = self.cluster_info
# Save both ACLU and CLU files.
save_clusters(self.filename_aclu, clusters)
save_clusters(self.filename_clu,
convert_to_clu(clusters, cluster_info['group']))
# Save CLUINFO and GROUPINFO files.
save_cluster_info(self.filename_acluinfo, cluster_info)
save_group_info(self.filename_groupinfo, self.group_info)
def save(self, renumber=False):
self.update_cluster_info()
self.update_group_info()
if renumber:
self.renumber()
clusters = get_array(self.clusters_renumbered)
cluster_info = self.cluster_info_renumbered
else:
clusters = get_array(self.clusters)
cluster_info = self.cluster_info
# Save both ACLU and CLU files.
save_clusters(self.filename_aclu, clusters)
save_clusters(self.filename_clu,
convert_to_clu(clusters, cluster_info['group']))
# Save CLUINFO and GROUPINFO files.
save_cluster_info(self.filename_acluinfo, cluster_info)
save_group_info(self.filename_groupinfo, self.group_info)
def renumber_clusters(clusters, cluster_info):
clusters_unique = get_array(get_indices(cluster_info))
nclusters = len(clusters_unique)
assert np.array_equal(clusters_unique, np.unique(clusters))
clusters_array = get_array(clusters)
groups = get_array(cluster_info["group"])
colors = get_array(cluster_info["color"])
groups_unique = np.unique(groups)
# Reorder clusters according to the group.
clusters_unique_reordered = np.hstack([sorted(clusters_unique[groups == group]) for group in groups_unique])
# WARNING: there's a +2 offset to avoid conflicts with the old convention
# cluster 0 = noise, cluster 1 = MUA.
clusters_renumbered = reorder(clusters_array, clusters_unique_reordered) + 2
cluster_permutation = reorder(clusters_unique_reordered, clusters_unique)
# Reorder cluster info.
groups_reordered = groups[cluster_permutation]
colors_reordered = colors[cluster_permutation]
# Recreate new cluster info.
cluster_info_reordered = pd.DataFrame(
{"color": colors_reordered, "group": groups_reordered}, dtype=np.int32, index=(np.arange(nclusters) + 2)
)
return clusters_renumbered, cluster_info_reordered
plt.rc("font", family="serif", size=14)
plt.rc("axes", labelsize=32)
plt.rc("xtick", labelsize=20)
plt.rc("ytick", labelsize=20)
plt.rc("legend", fontsize=24)
# Setup
fig = plt.figure(dpi=100, figsize=(6, 6))
ax = fig.add_subplot(111)
# Do stuff
for i in range(len(dags)):
data = dags[i]
label_index = 0
time = get_array(data, "copied voxels")
num_voxels = get_array(data, "num voxels")
#label = "(MISSING LABEL)"
#if -1 != names[i].find("nothread"):
# label = "Single-threaded"
#else:
# threadCount = re.search(r"_thread(\d+)", names[i]).group(1)
# label = "{} threads".format(threadCount)
kwargs = {"linestyle": "None", "marker": "o", "markersize": 5}
ax.set_xlabel("template voxels")
ax.set_ylabel("changed voxels")
ax.plot(num_voxels, time, **kwargs)
import matplotlib.pyplot as plt
import tools
from tools import get_array, dags, names, profiling_prompt
profiling_prompt()
plt.style.use("seaborn")
fig = plt.figure(dpi=100, figsize=(6, 8))
ax = fig.add_subplot(111)
for i in range(len(dags)):
data = dags[i]
paths = get_array(data, "paths")
colors = get_array(data, "colors")
kwargs = {"marker": "", "markersize": 2}
ax.set_xlabel("frames")
ax.set_ylabel("time (ms)")
#color = "red" if "headless=1" in names[i] else "green"
#ax.plot(tools.indices, paths, label="paths " + names[i], color=color, **kwargs)
#ax.plot(tools.indices, colors, label="colors " + names[i], color=color, **kwargs)
ax.plot(tools.indices, paths, label="paths " + names[i], **kwargs)
ax.plot(tools.indices, colors, label="colors " + names[i], **kwargs)
#ax.set_aspect( 1.0/ax.get_data_ratio()*4.0/3.0 );
leg = ax.legend()
#leg.set_in_layout(False);
plt.rc("font", family="serif", size=14)
plt.rc("axes", labelsize=32)
plt.rc("xtick", labelsize=20)
plt.rc("ytick", labelsize=20)
plt.rc("legend", fontsize=24)
# Setup
fig = plt.figure(dpi=100, figsize=(6, 6))
ax = fig.add_subplot(111)
# Do stuff
for i in range(len(dags)):
data = dags[i]
label_index = 0
time = get_array(data, "edits")
num_voxels = get_array(data, "num voxels")
label = "(MISSING LABEL)"
if -1 != names[i].find("normal"):
label = "Ref."
elif -1 != names[i].find("compempty"):
label = "Compr. (NE)"
elif -1 != names[i].find("decempty"):
label = "Dec. (NE)"
elif -1 != names[i].find("comp"):
label = "Compr. (Full)"
kwargs = {"linestyle": "None", "marker": "o", "markersize": 5}
ax.set_xlabel("template voxels")
import numpy as np
import matplotlib.pyplot as plt
from tools import get_array, dags, names
plots_map = {}
names = ["first pass", "start threads", "waiting", "second pass"]
names = [
"first pass", "leaf level edits", "high level edits", "bunch color copy",
"full color copy", "find_node", "add_node", "second pass"
]
# names = ["leaf level edits average", "high level edits average", "bunch color copy average", "find_or_add average"]
# names = ["find_or_add average"]
num_nodes = None
for data in dags:
new_num_nodes = get_array(data, "num nodes")
assert num_nodes is None or np.all(
num_nodes == new_num_nodes), "Edits are different!"
num_nodes = new_num_nodes
for name in names:
if name not in plots_map:
plots_map[name] = []
plots_map[name].append(get_array(data, name))
width = 0.3
indices = np.argsort(num_nodes)
plt.xlabel("nodes")
plt.ylabel("time (ms)")
import matplotlib
import numpy as np
from tools import get_array, dags, page_sizes
import matplotlib.pyplot as plt
edits_time = []
edits_num_nodes = []
for dag in dags:
edits_time.append(get_array(dag, "edits"))
edits_num_nodes.append(get_array(dag, "numNodes"))
kwargs = {"linestyle": "None", "marker": "o", "markersize": 5}
plt.xlabel("nodes")
plt.ylabel("time")
cmap = matplotlib.cm.get_cmap('Spectral')
log_page_sizes = np.log2(np.array(page_sizes))
log_page_sizes = (log_page_sizes - np.min(log_page_sizes)) / (
np.max(log_page_sizes) - np.min(log_page_sizes))
labels = set()
for i in range(len(dags)):
color = cmap(log_page_sizes[i])
label = page_sizes[i]
if label in labels:
label = ""
else:
labels.add(label)
plt.plot(edits_num_nodes[i],
edits_time[i],
import numpy as np
import matplotlib.pyplot as plt
from tools import get_array, dags, names
data = dags[0]
find_or_add_index = get_array(data, "find_or_add index")
find_or_add_time = get_array(data, "find_or_add add time")
find_or_add_add = get_array(data, "find_or_add is add")
find_or_add_level = get_array(data, "find_or_add level")
kwargs = {"linestyle": "None", "marker": "o", "markersize": 3}
plt.xlabel("index")
plt.ylabel("time")
indices_add = find_or_add_add == 1
indices_noadd = find_or_add_add == 0
print("Num add: ", np.sum(indices_add))
print("Num no add: ", np.sum(indices_noadd))
plt.title(names[0])
assert np.sum(indices_add) + np.sum(indices_noadd) == len(find_or_add_index)
plot_index = 0
for level in range(32):
level_indices = level == find_or_add_level
if np.sum(level_indices) == 0:
continue
from tools import get_array, dags, results_prompt, defines
results_prompt("page_size_memory_overhead")
for i in range(len(dags)):
data = dags[i]
virtual_size = get_array(data, "virtual_size")
allocated_size = get_array(data, "allocated_size")
print("page size: {}; used: {}; allocated: {}; wasted: {}".format(
defines[i]["PAGE_SIZE"], virtual_size[0], allocated_size[0],
allocated_size[0] - virtual_size[0]))
"second pass",
"early exit checks",
"leaf edit",
"find or add leaf",
"interior edit",
"find or add interior",
# "start threads",
# "waiting",
"entirely full - add colors",
"skip edit - copy colors"
]
num_nodes = None
total_num_nodes = []
for data in dags:
new_num_nodes = get_array(data, "num voxels")
assert num_nodes is None or np.all(
num_nodes == new_num_nodes), "Edits are different!"
num_nodes = new_num_nodes
total_num_nodes += list(num_nodes)
for name in names:
if name not in plots_map:
plots_map[name] = []
plots_map[name] += list(get_array(data, name))
total_num_nodes = np.array(total_num_nodes)
width = 0.5
def save(self, renumber=False):
# Report progress.
self.report_progress_save(1, 6)
self.update_cluster_info()
self.update_group_info()
# Renumber internal variables, knowing that in this case the file
# will be automatically reloaded right afterwards.
if renumber:
self.renumber()
self.clusters = self.clusters_renumbered
self.cluster_info = self.cluster_info_renumbered
self._update_data()
# Update the changes in the HDF5 tables.
self.spike_table.cols.cluster_manual[:] = get_array(self.clusters)
# Report progress.
self.report_progress_save(2, 6)
# Update the clusters table.
# --------------------------
# Add/remove rows to match the new number of clusters.
self._update_table_size(self.clusters_table,
len(self.get_clusters_unique()))
self.clusters_table.cols.cluster[:] = self.get_clusters_unique()
self.clusters_table.cols.group[:] = np.array(self.cluster_info['group'])
# Report progress.
self.report_progress_save(3, 6)
# Update the group table.
# -----------------------
# Add/remove rows to match the new number of clusters.
groups = get_array(get_indices(self.group_info))
self._update_table_size(
self.groups_table,
len(groups), )
self.groups_table.cols.group[:] = groups
self.groups_table.cols.name[:] = list(self.group_info['name'])
# Commit the changes on disk.
self.kwik.flush()
# Report progress.
self.report_progress_save(4, 6)
# Save the CLU file.
# ------------------
save_clusters(self.filename_clu,
convert_to_clu(self.clusters, self.cluster_info['group']))
# Report progress.
self.report_progress_save(5, 6)
# Update the KWA file.
# --------------------
kwa={}
kwa['shanks'] = {
shank: dict(
cluster_colors=self.cluster_info['color'],
group_colors=self.group_info['color'],
) for shank in self.shanks
}
write_kwa(self.filename_kwa, kwa)
# Report progress.
self.report_progress_save(6, 6)
plt.rc("ytick", labelsize=20)
plt.rc("legend", fontsize=24)
# Setup
fig = plt.figure(dpi=120, figsize=(10, 6))
ax = fig.add_subplot(111)
# Plot
data = dags[0]
plt.xlabel("Cumulative #edited voxels")
plt.ylabel("Memory usage (MB)")
kwargs = {"marker": "", "markersize": 5}
num_voxels = get_array(data, "num voxels")
voxels_indices = np.cumsum(num_voxels)
color_size = get_custom_array(data, "color_size")
undo_redo_size = get_custom_array(data, "color_size undo_redo")
ax.plot(voxels_indices,
color_size + undo_redo_size,
label="With history",
**kwargs)
ax.plot(voxels_indices, color_size, label="Without history", **kwargs)
ax.ticklabel_format(axis='x', style='sci', scilimits=(0, 0))
leg = plt.legend()
for legobj in leg.legendHandles:
legobj.set_linewidth(6.0)
from tools import get_array, dags
import matplotlib.pyplot as plt
data = dags[0]
edits_time = get_array(data, "edits")
edits_radius = get_array(data, "radius")
edits_num_findadd = get_array(data, "num find or add")
edits_num_iteratedvoxels = get_array(data, "num iterated leaf voxels")
edits_num_editedvoxels = get_array(data, "num edited leaf voxels")
edits_num_nodes = get_array(data, "num nodes")
kwargs = {"linestyle": "None", "marker": "o", "markersize": 5}
plt.xlabel("radius")
plt.ylabel("time")
plt.plot(edits_radius, edits_time, **kwargs)
plt.figure()
plt.xlabel("findadd")
plt.ylabel("time")
plt.plot(edits_num_findadd, edits_time, **kwargs)
plt.figure()
plt.xlabel("Edited nodes")
plt.ylabel("Time (ms)")
plt.plot(edits_num_nodes, edits_time, **kwargs)
plt.figure()
plt.xlabel("iterated voxels")
plt.ylabel("time")
from tools import get_array, dags
import matplotlib.pyplot as plt
data = dags[0]
edits_time = get_array(data, "edits")
edits_num_nodes = get_array(data, "num nodes")
kwargs = {"linestyle": "None", "marker": "o", "markersize": 5}
plt.xlabel("nodes")
plt.ylabel("time")
plt.plot(edits_num_nodes, edits_time, **kwargs)
if "rebuilding colors" in data[:, 1]:
colors_time = get_array(data, "rebuilding colors")
print("plotting rebuilding colors")
plt.plot(edits_num_nodes, edits_time + colors_time, color="red", **kwargs)
plt.show()
import numpy as np
from tools import get_array, dags, profiling_prompt
profiling_prompt()
data = dags[0]
num_voxels = get_array(data, "copied voxels")
print("{} copied voxels".format(np.sum(num_voxels)))
import tools
from tools import get_array, dags, profiling_prompt
import matplotlib.pyplot as plt
profiling_prompt()
data = dags[0]
virtual_size = get_array(data, "virtual_size")
plt.xlabel("Frame")
plt.ylabel("Memory usage (MB)")
kwargs = {"marker": "", "markersize": 5}
plt.plot(tools.indices, virtual_size, label="Virtual size", **kwargs)
gc_freed_memory = get_array(data, "GC freed memory leaf level")
plt.plot(tools.indices,
virtual_size - gc_freed_memory,
label="Virtual size after GC",
marker="x")
depth = 16
for i in range(depth - 1):
gc_freed_memory = get_array(data, "GC freed memory level " + str(i))
plt.plot(tools.indices,
virtual_size - gc_freed_memory,
label="Virtual size after GC level " + str(i),
**kwargs)
import tools
from tools import get_array, dags
import matplotlib.pyplot as plt
data = dags[0]
edits_num_nodes = get_array(data, "num nodes")
edits_num_leaf_nodes = get_array(data, "num leaf nodes")
kwargs = {"linestyle": "None", "marker": "o", "markersize": 5}
plt.xlabel("frame")
plt.ylabel("nodes")
plt.plot(tools.indices, edits_num_nodes, label="num nodes", **kwargs)
plt.plot(tools.indices, edits_num_leaf_nodes, label="num leaf nodes", **kwargs)
plt.legend()
plt.show()