def _get_clustered_data_set(data_set: DataSet, clusters: list) -> DataSet:
"""
Combine vectors within each cluster to a single
data point and corresponding metadata structure.
Use mean vector as the cluster's representative
and all metadata structures excluding duplicates.
:param data_set: DataSet - Data points
and corresponding metadata structures.
:param clusters: list(list) - Data point
indices contained in each cluster.
:return: DataSet - Clustered data points
and corresponding metadata structures.
"""
# Create lists to store compressed vectors
# and corresponding metadata structures.
new_data_points = list()
new_metadata = list()
for index_list in clusters:
# Compress data points in current cluster
# by computing corresponding mean vector.
current_data_points = [
data_set.get_vector_at_index(i) for i in index_list
]
mean_vector = calculation.get_mean(current_data_points)
new_data_points.append(mean_vector)
# Compress metadata structures in current
# cluster by using corresponding merge function.
current_metadata_structure = set()
for i in index_list:
current_metadata_structure = DataSet.merge_metadata(
current_metadata_structure, data_set.get_metadata_at_index(i))
new_metadata.append(current_metadata_structure)
# Return compressed data.
return DataSet(new_data_points, new_metadata)
def _chinese_whispers_clustering(data_set: DataSet,
distance_threshold: float,
num_iterations: int = 30) -> DataSet:
"""
Cluster given data set in a way that only a
single data point within groups of nearby points
is kept. Use a temporary graph structure with
one node per data point and add an unweighted
edge between every pair of nodes with specified
maximum distance. Search for groups of nodes
with many pairwise edges using chinese whispers
graph clustering. This algorithm works in three
steps:
1. Assign each node a different label.
2. Iterate through all nodes assigning
each node the label used for the
majority of its neighbors.
3. Repeat second step a specified number
of times (use 30 iterations as
indicated in original paper).
See Chris Biemann: "Chinese Whispers - an Efficient Graph
Clustering Algorithm and its Application to Natural
Language Processing Problems" (2006) for details.
:param data_set: DataSet - Data points
and corresponding metadata structures.
:param distance_threshold: float - Maximum
distance between two data points in
order to consider them as candidates
for the same cluster.
:param num_iterations: int - Maximum number of
passes through all data points (default: 30).
:return: DataSet - Clustered data points
and corresponding metadata structures.
Compare with dlib implementation:
http://dlib.net/python/index.html#dlib.chinese_whispers_clustering
"""
# Create adjacency list representation
# of given data set only considering
# edges connecting data points within
# specified distance threshold.
edges = [[] for _i in range(len(data_set))]
for i in range(len(data_set)):
for j in range(i + 1, len(data_set)):
# Get distance from vector at
# index i to vector at index j.
distance = calculation.get_distance(
data_set.get_vector_at_index(i),
data_set.get_vector_at_index(j))
# Compare current distance to specified threshold.
if numpy.less_equal(distance, distance_threshold):
edges[i].append(j)
edges[j].append(i)
# Initialize clustering labels so that every
# data point is assigned a different cluster.
labels = [i for i in range(len(data_set))]
# Initialize list to specify current data
# point sequence, i.e. the order in which
# data point labels are adjusted.
sequence = [i for i in range(len(data_set))]
for _i in range(num_iterations):
# Create flag to track if
# any label was adjusted.
label_adjusted = False
# Create random ordering of data points.
numpy.random.shuffle(sequence)
# Iterate through data points in this order.
label_counter = dict()
for current_index in sequence:
# Count all occurrences of the
# same label in direct neighborhood
# of currently chosen data point.
label_counter.clear()
for neighbor_index in edges[current_index]:
neighbor_label = labels[neighbor_index]
if neighbor_label in label_counter:
label_counter[neighbor_label] += 1
else:
label_counter[neighbor_label] = 1
# Get the most common label.
best_label = -1
best_count = -1
for label, count in label_counter.items():
if count > best_count:
best_label = label
best_count = count
# Assign most common label to current data
# point only if it exists and is different.
if (best_label >= 0) and (best_label != labels[current_index]):
labels[current_index] = best_label
label_adjusted = True
# Check if any label was adjusted. If not, the
# process has converged and can be stopped.
if not label_adjusted:
break
# Reconstruct corresponding clusters.
clusters = dict()
for i, label in enumerate(labels):
if label in clusters:
clusters[label].append(i)
else:
clusters[label] = [i]
clusters = list(clusters.values())
# Return data set containing
# one representative description
# vector with corresponding metadata
# structures for each cluster.
return _get_clustered_data_set(data_set, clusters)