def test_equality(self):
graph = CachedDistanceGraph(nodes=[1, 2, 3, 4],
distance=self.distance_cls(),
symmetric=True)
io_graph_a = DenGraphIO(base_graph=graph,
cluster_distance=5,
core_neighbours=5)
io_graph_b = DenGraphIO(base_graph=graph,
cluster_distance=5,
core_neighbours=5)
self.assertEqual(io_graph_a, io_graph_b)
io_graph_b = DenGraphIO(base_graph=graph,
cluster_distance=6,
core_neighbours=5)
self.assertNotEqual(io_graph_a, io_graph_b)
io_graph_b = DenGraphIO(base_graph=graph,
cluster_distance=5,
core_neighbours=6)
self.assertNotEqual(io_graph_a, io_graph_b)
io_graph_a[5] = {}
io_graph_a[6] = {}
io_graph_b = DenGraphIO(base_graph=CachedDistanceGraph(
nodes=[1, 2, 3, 4], distance=self.distance_cls(), symmetric=True),
cluster_distance=5,
core_neighbours=5)
io_graph_b[5] = {}
io_graph_b[6] = {}
self.assertEqual(io_graph_a, io_graph_b)
io_graph_b[12] = {}
self.assertNotEqual(io_graph_a, io_graph_b)
def test_remove_edge_to_delete_cluster(self):
literal = textwrap.dedent("""
1,2,3,4,5,6
0,1,1,1,1,1
1,0,0,0,0,0
1,0,0,0,0,0
1,0,0,0,0,0
1,0,0,0,0,0
1,0,0,0,0,0
""".strip())
validation_literal = textwrap.dedent("""
1,2,3,4,5,6
0,1,1,1,1,0
1,0,0,0,0,0
1,0,0,0,0,0
1,0,0,0,0,0
1,0,0,0,0,0
0,0,0,0,0,0
""".strip())
io_graph = DenGraphIO(
base_graph=dengraph.graphs.graph_io.csv_graph_reader(
literal.splitlines(), symmetric=True),
cluster_distance=1,
core_neighbours=5)
del io_graph["1":"6"]
validation_io_graph = DenGraphIO(
base_graph=dengraph.graphs.graph_io.csv_graph_reader(
validation_literal.splitlines(), symmetric=True),
cluster_distance=1,
core_neighbours=5)
self.assertEqual(validation_io_graph, io_graph)
def setUp(self):
self.io_graph = DenGraphIO(base_graph=DistanceGraph(
nodes=[1, 2, 3, 4], distance=DeltaDistance(), symmetric=True),
cluster_distance=1,
core_neighbours=5)
self.one_cluster_graph = DenGraphIO(base_graph=DistanceGraph(
nodes=[(1, 1), (1, 2), (1, 3), (1, 4), (1, 5), (1, 6), (2, 1),
(2, 2), (2, 3), (2, 4), (2, 5), (2, 6)],
distance=ListDistance(),
symmetric=True),
cluster_distance=.1,
core_neighbours=5)
def test_split_to_several_clusters(self):
literal = textwrap.dedent("""
1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17
0,1,0,0,0,1,0,0,0, 1, 1, 0, 0, 0, 0, 0, 0
1,0,1,1,1,0,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
0,1,0,0,0,0,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
0,1,0,0,0,0,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
0,1,0,0,0,0,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
1,0,0,0,0,0,1,1,1, 0, 0, 0, 0, 0, 0, 0, 0
0,0,0,0,0,1,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
0,0,0,0,0,1,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
0,0,0,0,0,1,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
1,0,0,0,0,0,0,0,0, 0, 0, 1, 1, 1, 0, 0, 1
1,0,0,0,0,0,0,0,0, 0, 0, 1, 0, 0, 1, 1, 1
0,0,0,0,0,0,0,0,0, 1, 1, 0, 0, 0, 0, 0, 0
0,0,0,0,0,0,0,0,0, 1, 0, 0, 0, 0, 0, 0, 0
0,0,0,0,0,0,0,0,0, 1, 0, 0, 0, 0, 0, 0, 0
0,0,0,0,0,0,0,0,0, 0, 1, 0, 0, 0, 0, 0, 0
0,0,0,0,0,0,0,0,0, 0, 1, 0, 0, 0, 0, 0, 0
0,0,0,0,0,0,0,0,0, 1, 1, 0, 0, 0, 0, 0, 0
""".strip())
validation_literal = textwrap.dedent("""
2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17
0,1,1,1,0,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
1,0,0,0,0,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
1,0,0,0,0,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
1,0,0,0,0,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
0,0,0,0,0,1,1,1, 0, 0, 0, 0, 0, 0, 0, 0
0,0,0,0,1,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
0,0,0,0,1,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
0,0,0,0,1,0,0,0, 0, 0, 0, 0, 0, 0, 0, 0
0,0,0,0,0,0,0,0, 0, 0, 1, 1, 1, 0, 0, 1
0,0,0,0,0,0,0,0, 0, 0, 1, 0, 0, 1, 1, 1
0,0,0,0,0,0,0,0, 1, 1, 0, 0, 0, 0, 0, 0
0,0,0,0,0,0,0,0, 1, 0, 0, 0, 0, 0, 0, 0
0,0,0,0,0,0,0,0, 1, 0, 0, 0, 0, 0, 0, 0
0,0,0,0,0,0,0,0, 0, 1, 0, 0, 0, 0, 0, 0
0,0,0,0,0,0,0,0, 0, 1, 0, 0, 0, 0, 0, 0
0,0,0,0,0,0,0,0, 1, 1, 0, 0, 0, 0, 0, 0
""".strip())
io_graph = DenGraphIO(
base_graph=dengraph.graphs.graph_io.csv_graph_reader(
literal.splitlines(), symmetric=True),
cluster_distance=1,
core_neighbours=3)
validation_io_graph = DenGraphIO(
base_graph=dengraph.graphs.graph_io.csv_graph_reader(
validation_literal.splitlines(), symmetric=True),
cluster_distance=1,
core_neighbours=3)
del io_graph["1"]
self.assertEqual(validation_io_graph, io_graph)
def test_simple_noise(self):
io_graph = DenGraphIO(base_graph=CachedDistanceGraph(
nodes=[1, 2, 3, 4, 5, 6],
distance=self.distance_cls(),
symmetric=True),
core_neighbours=5,
cluster_distance=5)
self.assertEqual(1, len(io_graph.clusters))
self.assertEqual(set(), io_graph.noise)
io_graph._recluster(io_graph.clusters[0])
self.assertEqual(1, len(io_graph.clusters))
self.assertEqual(set(), io_graph.noise)
def test_real_world_example(self):
class DistanceMatrixDistance(object):
is_symmetric = True
matrix = []
def __call__(self, a, b):
return self.matrix[a - 1][b - 1]
file_path = os.path.join(os.path.dirname(dengraph_unittests.__file__),
"data/tree_distances.csv.zip")
distance = DistanceMatrixDistance()
nodes = None
with zipfile.ZipFile(file_path) as zipped_data:
with zipped_data.open("tree_distances.csv") as tree_file:
if sys.version_info >= (3, ):
tree_file = io.TextIOWrapper(tree_file)
csvreader = csv.reader(tree_file, delimiter=",")
header_initialized = False
for row in csvreader:
try:
if row[0].startswith("#"):
continue
except IndexError:
pass
if not header_initialized:
nodes = [int(element) for element in row]
header_initialized = True
continue
distance.matrix.append([float(element) for element in row])
dengraph = DenGraphIO(base_graph=CachedDistanceGraph(nodes=nodes,
distance=distance,
symmetric=True),
core_neighbours=5,
cluster_distance=.00001)
print(len(dengraph.clusters))
def test_zero_clusters(self):
io_graph = DenGraphIO(base_graph=DistanceGraph(
nodes=[1, 2, 3, 4], distance=DeltaDistance(), symmetric=True),
cluster_distance=1,
core_neighbours=5)
with self.assertRaises(ValueError):
davies_bouldin_score(io_graph.clusters, io_graph.graph)
def test_noise(self):
graph = CachedDistanceGraph(nodes=[1, 2, 3, 4, 5, 6, 20],
distance=self.distance_cls(),
symmetric=True)
io_graph = DenGraphIO(base_graph=graph,
cluster_distance=5,
core_neighbours=5)
self.assertEqual(set([20]), io_graph.noise)
def test_simple_graph(self):
graph = CachedDistanceGraph(nodes=self.random_nodes(100, 10) +
self.random_nodes(100, 40),
distance=self.distance_cls(),
symmetric=True)
io_graph = DenGraphIO(base_graph=graph,
cluster_distance=5,
core_neighbours=5)
self.assertEqual(2, len(io_graph.clusters))
def test_two_cluster(self):
io_graph = DenGraphIO(base_graph=DistanceGraph(
nodes=[1, 2, 3, 4, 5, 6, 13, 14, 15, 16, 17, 18],
distance=DeltaDistance(),
symmetric=True),
cluster_distance=5,
core_neighbours=5)
self.assertEqual(
.25, davies_bouldin_score(io_graph.clusters, io_graph.graph))
def test_two_cluster(self):
io_graph = DenGraphIO(
base_graph=DistanceGraph(
nodes=[(1,1), (1,2), (1,3), (1,4), (1,5), (1,6), (2,1), (2,2), (2,3), (2,4), (2,5), (2,6)],
distance=ListDistance(),
symmetric=True),
cluster_distance=.1,
core_neighbours=5
)
self.assertEqual(1.0, silhouette_score(io_graph.clusters, io_graph.graph))
def test_one_cluster(self):
io_graph = DenGraphIO(
base_graph=DistanceGraph(
nodes=[1, 2, 3, 4, 5, 6],
distance=DeltaDistance(),
symmetric=True),
cluster_distance=5,
core_neighbours=5
)
self.assertEqual(-1, silhouette_score(io_graph.clusters, io_graph.graph))
def test_two_cluster_with_zero_distance(self):
io_graph = DenGraphIO(
base_graph=DistanceGraph(
nodes=[(1,1), (1,2), (1,3), (1,4), (1,5), (1,6), (2,1), (2,2), (2,3), (2,4), (2,5), (2,6)],
distance=ListDistance(),
symmetric=True),
cluster_distance=.1,
core_neighbours=5
)
self.assertEqual(float("inf"), calinski_harabasz_score(io_graph.clusters, io_graph.graph))
def test_two_cluster(self):
io_graph = DenGraphIO(
base_graph=DistanceGraph(
nodes=[1, 2, 3, 4, 5, 6, 13, 14, 15, 16, 17, 18],
distance=DeltaDistance(),
symmetric=True),
cluster_distance=5,
core_neighbours=5
)
self.assertAlmostEqual(123.4, calinski_harabasz_score(io_graph.clusters, io_graph.graph), 1)
def _validation_graph_for_nodes(self,
distance,
nodes,
cluster_distance,
core_neighbours,
graph_type=CachedDistanceGraph):
graph = graph_type(nodes=nodes, distance=distance(), symmetric=True)
return DenGraphIO(base_graph=graph,
cluster_distance=cluster_distance,
core_neighbours=core_neighbours)
def perform_precalculated_clustering(ctx, eta, epsilon):
results = {}
if ctx.obj.get("use_input", False):
configuration = ctx.obj.get("configurations", None)[0]
distance = configuration.get("distances", [None])[0]
structure = ctx.obj.get("structure", None)
file_path = structure.input_file_path(
file_type="csv") # expecting csv file
graph = _create_graph(ctx, file_path)
for single_eta in eta:
for single_epsilon in epsilon:
start = time.time()
clustering = DenGraphIO(base_graph=graph,
cluster_distance=single_epsilon,
core_neighbours=single_eta)
end = time.time()
cluster_distance = ClusterDistance(distance=distance())
clustering.graph.distance = cluster_distance
print(
"---> performed clustering with eta %s and epsilon %s in %s"
% (single_eta, single_epsilon, end - start))
results.setdefault("results", []).append({})
current_result = results["results"][-1]
current_result.setdefault(
"meta", {})["algorithm"] = clustering.__class__.__name__
current_result.setdefault("meta", {})["eta"] = single_eta
current_result.setdefault("meta",
{})["epsilon"] = single_epsilon
current_result["duration"] = end - start
for cluster_idx, cluster in enumerate(clustering):
current_result.setdefault("clusters", []).append(
[node.key for node in cluster]) # TODO: determine CR
print("[cluster %s] %s" % (cluster_idx, len(cluster)))
print("[noise] %s" % len(clustering.noise))
for noise in clustering.noise:
current_result.setdefault("noise", []).append(noise.key)
# for score in [silhouette_score, calinski_harabasz_score, davies_bouldin_score]:
# for score in [silhouette_score]:
for score in []:
try:
the_score = score(clustering.clusters,
clustering.graph)
except ValueError:
the_score = None
current_result.setdefault("scores",
{})[score.__name__] = the_score
print("Got a %s of %s" % (score.__name__, the_score))
output_results(
ctx=ctx,
results=results,
version=determine_version(os.path.dirname(assess_workflows.__file__)),
source="%s (%s)" % (__file__, "perform_precalculated_clustering"))
def test_node_not_in_cluster(self):
io_graph = DenGraphIO(
base_graph=DistanceGraph(
nodes=[(1,1), (1,2), (1,3), (1,4), (1,5), (1,6), (2,1)],
distance=ListDistance(),
symmetric=True),
cluster_distance=.1,
core_neighbours=5
)
with self.assertRaises(dengraph.graph.NoSuchNode):
avg_inter_cluster_distance((2,1), io_graph.clusters[0], io_graph.graph)
def test_one_cluster(self):
io_graph = DenGraphIO(
base_graph=DistanceGraph(
nodes=[1, 2, 3, 4, 5, 6],
distance=DeltaDistance(),
symmetric=True),
cluster_distance=5,
core_neighbours=5
)
with self.assertRaises(ValueError):
calinski_harabasz_score(io_graph.clusters, io_graph.graph)
def test_containment(self):
io_graph = DenGraphIO(base_graph=CachedDistanceGraph(
nodes=[1, 2, 3, 4], distance=self.distance_cls(), symmetric=True),
cluster_distance=5,
core_neighbours=5)
self.assertFalse(1 in io_graph)
self.assertFalse(slice(1, 2) in io_graph)
io_graph[5] = {}
io_graph[6] = {}
self.assertTrue(1 in io_graph)
self.assertTrue(slice(1, 2) in io_graph)
def clusterer(self):
if self._clusterer is None:
distance = ClusterDistance(distance=self.distance)
self._clusterer = DenGraphIO(
base_graph=AdjacencyGraph(
DistanceGraph(nodes=self._nodes,
distance=distance,
symmetric=True)),
cluster_distance=self.cluster_distance,
core_neighbours=self.core_neighbours)
self._clusterer.graph.distance = distance
return self._clusterer
def test_several_cluster(self):
node_list = []
for node in range(10):
for position in range(10):
node_list.append((node, position))
io_graph = DenGraphIO(base_graph=DistanceGraph(nodes=node_list,
distance=ListDistance(),
symmetric=True),
cluster_distance=.1,
core_neighbours=5)
self.assertEqual(
0.0, davies_bouldin_score(io_graph.clusters, io_graph.graph))
def test_remove_border_on_core(self):
literal = textwrap.dedent("""
1,2,3,4,5,6,7,8,9,10,11
0,1,1,1,1,1,0,0,0,0,1
1,0,0,0,0,0,1,1,1,1,0
1,0,0,0,0,0,0,0,0,0,0
1,0,0,0,0,0,0,0,0,0,0
1,0,0,0,0,0,0,0,0,0,0
1,0,0,0,0,0,0,0,0,0,0
0,1,0,0,0,0,0,0,0,0,0
0,1,0,0,0,0,0,0,0,0,0
0,1,0,0,0,0,0,0,0,0,0
0,1,0,0,0,0,0,0,0,0,0
1,0,0,0,0,0,0,0,0,0,0
""".strip())
validation_literal = textwrap.dedent("""
1,3,4,5,6,7,8,9,10,11
0,1,1,1,1,0,8,9,0,1
1,0,0,0,0,0,0,0,0,0
1,0,0,0,0,0,0,0,0,0
1,0,0,0,0,0,0,0,0,0
1,0,0,0,0,0,0,0,0,0
0,0,0,0,0,0,0,0,0,0
0,0,0,0,0,0,0,0,0,0
0,0,0,0,0,0,0,0,0,0
0,0,0,0,0,0,0,0,0,0
1,0,0,0,0,0,0,0,0,0
""".strip())
graph = dengraph.graphs.graph_io.csv_graph_reader(literal.splitlines(),
symmetric=True)
io_graph = DenGraphIO(base_graph=graph,
cluster_distance=1,
core_neighbours=5)
del io_graph["2"]
validation_io_graph = DenGraphIO(
base_graph=dengraph.graphs.graph_io.csv_graph_reader(
validation_literal.splitlines(), symmetric=True),
cluster_distance=1,
core_neighbours=5)
self.assertEqual(validation_io_graph, io_graph)
def validate_representatives(ctx, eta, epsilon, threshold):
"""
Method performs a clustering and from this resulting clusters, calculates the cluster
representatives to measure our current current representation. For validation of the cluster
representatives we evaluate the distance of each clustered object to the calculated cluster
representative. Whenever the distance is bigger than the given epsilon for clustering, we
have a bad representation.
:param ctx:
:return:
"""
if ctx.obj.get("use_input", False):
configuration = ctx.obj.get("configurations", None)[0]
distance_cls = configuration.get("distances", [None])[0]
structure = ctx.obj.get("structure", None)
file_path = structure.input_file_path(
file_type="csv") # expecting csv file
graph = _create_graph(ctx, file_path)
clustering = DenGraphIO(base_graph=graph,
cluster_distance=epsilon,
core_neighbours=eta)
cluster_distance = ClusterDistance(distance=distance_cls(),
threshold=threshold)
clustering.graph.distance = cluster_distance
results = {
"meta": {
"eta": eta,
"epsilon": epsilon,
"threshold": threshold
},
"clusters": {}
}
# calculate CRs from clusters
for cluster_index, cluster in enumerate(clustering):
cluster_representative = cluster_distance.mean(list(cluster))
for tree_object in cluster:
# calculate distance to cluster representative
distance = cluster_distance(cluster_representative,
tree_object)
results["clusters"].setdefault(cluster_index, {}).setdefault(
"tree", []).append(tree_object.key)
results["clusters"].setdefault(cluster_index, {}).setdefault(
"distance", []).append(distance)
output_results(ctx=ctx,
results=results,
version=determine_version(
os.path.dirname(assess_workflows.__file__)),
source="%s (%s)" %
(__file__, "validate_representatives"))
def test_remove_node_border_connection(self):
"""
validate that a cluster is also split when it is only connected by a border node
"""
literal = textwrap.dedent("""
1,2,3,4,5,6,7,8,9
0,1,0,0,0,0,1,0,0
1,0,1,1,1,0,0,0,0
0,1,0,0,0,0,0,0,0
0,1,0,0,0,0,0,0,0
0,1,0,0,0,1,1,0,0
0,0,0,0,1,0,0,0,0
1,0,0,0,1,0,0,1,1
0,0,0,0,0,0,1,0,0
0,0,0,0,0,0,1,0,0
""".strip())
validation_literal = textwrap.dedent("""
1,2,3,4,6,7,8,9
0,1,0,0,0,1,0,0
1,0,1,1,0,0,0,0
0,1,0,0,0,0,0,0
0,1,0,0,0,0,0,0
0,0,0,0,0,0,0,0
1,0,0,0,0,0,1,1
0,0,0,0,0,1,0,0
0,0,0,0,0,1,0,0
""".strip())
io_graph = DenGraphIO(
base_graph=dengraph.graphs.graph_io.csv_graph_reader(
literal.splitlines(), symmetric=True),
cluster_distance=1,
core_neighbours=3)
validation_io_graph = DenGraphIO(
base_graph=dengraph.graphs.graph_io.csv_graph_reader(
validation_literal.splitlines(), symmetric=True),
cluster_distance=1,
core_neighbours=3)
del io_graph["5"]
self.assertEqual(validation_io_graph, io_graph)
def test_get(self):
io_graph = DenGraphIO(base_graph=CachedDistanceGraph(
nodes=[1, 2, 3, 4], distance=self.distance_cls(), symmetric=True),
cluster_distance=5,
core_neighbours=5)
with self.assertRaises(dengraph.graph.NoSuchNode):
io_graph[1]
with self.assertRaises(dengraph.graph.NoSuchEdge):
io_graph[1:2]
io_graph[5] = {}
io_graph[6] = {}
self.assertEqual(1, io_graph[1:2])
with self.assertRaises(dengraph.graph.NoSuchEdge):
io_graph[1:7]
def test_iteration(self):
io_graph = DenGraphIO(base_graph=CachedDistanceGraph(
nodes=[1, 2, 3, 4], distance=self.distance_cls(), symmetric=True),
cluster_distance=5,
core_neighbours=5)
with self.assertRaises(StopIteration):
next(iter(io_graph))
io_graph[5] = {}
io_graph[6] = {}
io_graph[12] = {}
node_set = set()
for cluster in io_graph:
for node in cluster:
node_set.add(node)
self.assertEqual(set([1, 2, 3, 4, 5, 6]), node_set)
def test_remove_incremental_behaviour(self):
base_nodes = [1, 2, 3, 4, 5, 6, 12, 13, 14, 15, 16, 17]
remove_nodes = [7]
validation_io_graph = self._validation_graph_for_nodes(
nodes=base_nodes,
distance=self.distance_cls,
cluster_distance=5,
core_neighbours=5)
graph = CachedDistanceGraph(nodes=base_nodes + remove_nodes,
distance=self.distance_cls(),
symmetric=True)
io_graph = DenGraphIO(base_graph=graph,
cluster_distance=5,
core_neighbours=5)
for node in remove_nodes:
del io_graph[node]
self.assertEqual(validation_io_graph, io_graph)
def test_remove_cluster(self):
base_nodes = [1, 2, 3, 4, 5]
remove_nodes = [6]
validation_io_graph = self._validation_graph_for_nodes(
nodes=base_nodes,
distance=self.distance_cls,
cluster_distance=5,
core_neighbours=5)
io_graph = DenGraphIO(base_graph=CachedDistanceGraph(
nodes=base_nodes + remove_nodes,
distance=self.distance_cls(),
symmetric=True),
cluster_distance=5,
core_neighbours=5)
for node in remove_nodes:
del io_graph[node]
self.assertEqual(validation_io_graph, io_graph)
def test_add_incremental_behaviour(self):
base_nodes = [1, 2, 3, 4, 5, 6, 7, 8]
nodes_to_add = [10]
validation_io_graph = self._validation_graph_for_nodes(
nodes=base_nodes + nodes_to_add,
distance=self.distance_cls,
cluster_distance=5,
core_neighbours=5)
graph = CachedDistanceGraph(nodes=base_nodes,
distance=self.distance_cls(),
symmetric=True)
io_graph = DenGraphIO(base_graph=graph,
cluster_distance=5,
core_neighbours=5)
for node in nodes_to_add:
io_graph[node] = {}
self.assertEqual(validation_io_graph, io_graph)
def test_noise_removal(self):
base_nodes = [1, 2, 3, 4, 5, 6, 7, 8]
remove_nodes = [30, 31]
validation_io_graph = self._validation_graph_for_nodes(
nodes=base_nodes,
distance=self.distance_cls,
cluster_distance=5,
core_neighbours=5)
graph = CachedDistanceGraph(nodes=base_nodes + remove_nodes,
distance=self.distance_cls(),
symmetric=True)
io_graph = DenGraphIO(base_graph=graph,
cluster_distance=5,
core_neighbours=5)
self.assertTrue(all([node in io_graph.noise for node in remove_nodes]))
for node in remove_nodes:
del io_graph[node]
self.assertEqual(validation_io_graph, io_graph)
self.assertTrue(
all([node not in io_graph.noise for node in remove_nodes]))
