def test_same_attributes_different_count(self):
tree_1 = Prototype()
root = tree_1.add_node("root", pid=1, ppid=0, tme=0, exit_tme=0)
for _ in range(5):
root.add_node("node", pid=2, ppid=1, tme=0, exit_tme=0)
tree_2 = Prototype()
root = tree_2.add_node("root", pid=1, ppid=0, tme=0, exit_tme=0)
for _ in range(35):
root.add_node("node", pid=2, ppid=1, tme=0, exit_tme=0)
signature = ParentChildByNameTopologySignature()
algorithm = IncrementalDistanceAlgorithm(
signature=signature,
distance=lambda **kwargs: StartExitDistance(weight=0, **kwargs),
cache_statistics=SplittedStatistics)
algorithm.prototypes = [tree_1, tree_2]
decorator = DistanceMatrixDecorator(normalized=False)
decorator.wrap_algorithm(algorithm)
algorithm.start_tree()
for event in tree_1.event_iter(supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
algorithm.start_tree()
for event in tree_2.event_iter(supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
data = decorator.data()
self.assertEqual(data[0][0][1], data[1][0][0])
def test_simple(self):
decorator = AnomalyDecorator()
algorithm = IncrementalDistanceAlgorithm()
algorithm.prototypes = [simple_prototype()]
decorator.wrap_algorithm(algorithm)
algorithm.start_tree()
for event in Event.from_tree(simple_monitoring_tree(), supported={ProcessStartEvent: True}):
algorithm.add_event(event)
algorithm.finish_tree()
self.assertEqual(decorator.data(), [
[
[
[False, False, False, False, False]
]
]
])
algorithm.start_tree()
for event in Event.from_tree(simple_monitoring_tree(), supported={ProcessStartEvent: True}):
algorithm.add_event(event)
algorithm.finish_tree()
self.assertEqual(decorator.data(), [
[
[
[False, False, False, False, False]
]
],
[
[
[False, False, False, False, False]
]
]
])
def test_external(self):
def distance(**kwargs):
distance = StartExitDistance(weight=0, **kwargs)
distance.supported[TrafficEvent] = True
return distance
algorithm = IncrementalDistanceAlgorithm(
signature=ParentChildByNameTopologySignature(),
distance=distance,
cache_statistics=SplittedStatistics)
decorator = DistanceMatrixDecorator(normalized=False)
decorator.wrap_algorithm(algorithm)
the_tree = real_tree(
path="data/c01-007-102/2/1146-2-process.csv",
absolute=True
)
algorithm.prototypes = [the_tree]
algorithm.start_tree()
for event in the_tree.event_iter(supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
print(decorator.data())
self.assertTrue(False)
def test_ensemble_with_prototypes(self):
decorator = AnomalyDecorator()
algorithm = IncrementalDistanceAlgorithm(
signature=EnsembleSignature(
signatures=[ParentChildByNameTopologySignature(),
ParentChildOrderByNameTopologySignature()]))
algorithm.prototypes = [simple_prototype(), simple_monitoring_tree()]
decorator.wrap_algorithm(algorithm)
algorithm.start_tree()
for event in Event.from_tree(simple_monitoring_tree(), supported={ProcessStartEvent: True}):
algorithm.add_event(event)
algorithm.finish_tree()
self.assertEqual(decorator.data(), [
[
[
[False, False, False, False, False],
[False, False, False, False, False]
],
[
[False, False, False, False, True],
[False, False, False, False, False]
]
]
])
def test_normalized_results(self):
decorator = DistanceDecorator(normalized=True)
algorithm = IncrementalDistanceAlgorithm(
signature=EnsembleSignature(
signatures=[ParentChildByNameTopologySignature(),
ParentChildOrderByNameTopologySignature()])
)
algorithm.prototypes = [simple_prototype(), simple_monitoring_tree()]
decorator.wrap_algorithm(algorithm)
algorithm.start_tree()
self.assertEqual([[[[], []], [[], []]]], decorator.data())
for event in Event.from_tree(simple_monitoring_tree(), supported={ProcessStartEvent: True}):
algorithm.add_event(event)
algorithm.finish_tree()
self.assertEqual([[
[ # ParentChildByNameTopologySignature
[2 / 3, 1 / 3, 1 / 3, 0.0], # simple_prototype
[2 / 3, 1 / 3, 1 / 3, 0.0] # simple_monitoring_tree
],
[ # ParentChildOrderByNameTopologySignature
[4 / 5, 3 / 5, 3 / 5, 2 / 5],
[2 / 3, 1 / 3, 1 / 3, 0.0]
]]], decorator.data())
def test_ensemble(self):
def distance(**kwargs):
distance = StartExitDistance(weight=.5, **kwargs)
distance.supported[TrafficEvent] = False
return distance
signature = EnsembleSignature(
signatures=[
ParentChildByNameTopologySignature(), ParentSiblingSignature(width=2)])
algorithm = IncrementalDistanceAlgorithm(
signature=signature,
distance=distance,
cache_statistics=SetStatistics
)
decorator = DistanceMatrixDecorator(normalized=True)
decorator.wrap_algorithm(algorithm)
algorithm.prototypes = [simple_prototype()]
algorithm.start_tree()
for event in simple_monitoring_tree().event_iter(supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
print(decorator.data())
self.assertTrue(False)
def test_negative_values(self):
def distance_builder(**kwargs):
distance = SimpleDistance(**kwargs)
return distance
tree_one = real_tree(path="data/c01-007-102/2/1078-2-process.csv",
absolute=True)
tree_two = real_tree(path="data/c01-007-102/2/1165-2-process.csv",
absolute=True)
signature = ParentChildByNameTopologySignature()
algorithm = IncrementalDistanceAlgorithm(
signature=signature,
distance=distance_builder,
cache_statistics=SetStatistics)
algorithm.prototypes = [tree_one, tree_two]
decorator = DistanceMatrixDecorator(normalized=True)
decorator.wrap_algorithm(algorithm)
algorithm.start_tree()
for event in tree_one.event_iter(supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
algorithm.start_tree()
for event in tree_two.event_iter(supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
print(decorator.data())
self.assertTrue(False)
def test_parameter_distance(self):
prototype = Prototype()
root = prototype.add_node("root", tme=0, exit_tme=0, pid=1, ppid=0, param=1)
for i in range(5):
root.add_node("child_%d" % i, tme=0, exit_tme=0, pid=i + 2, ppid=1, param=1)
next(root.children()).add_node("child", tme=0, exit_tme=0, pid=8, ppid=2, param=1)
tree = Prototype()
root = tree.add_node("root", tme=0, exit_tme=0, pid=1, ppid=0, param=1)
for i in range(5):
root.add_node("child_%d" % i, tme=0, exit_tme=0, pid=i + 2, ppid=1, param=4)
next(root.children()).add_node("child", tme=0, exit_tme=0, pid=8, ppid=2, param=4)
for weight, result in [(1, 0), (.5, 6), (0, 12)]:
def distance(**kwargs):
distance = StartExitDistance(weight=weight, **kwargs)
distance.supported = {
ProcessStartEvent: True,
ProcessExitEvent: True,
ParameterEvent: True
}
return distance
signature = EnsembleSignature(signatures=[ParentChildByNameTopologySignature()])
algorithm = IncrementalDistanceAlgorithm(
signature=signature,
distance=distance,
cache_statistics=SetStatistics
)
decorator = DistanceMatrixDecorator(normalized=False)
decorator.wrap_algorithm(algorithm)
algorithm.prototypes = [prototype]
algorithm.start_tree()
algorithm.add_events(tree.event_iter(supported=algorithm.supported))
algorithm.finish_tree()
self.assertEqual(result, decorator.data()[0][0][0])
def test_real_trees(self):
def distance_builder(**kwargs):
distance = StartExitDistance()
distance.supported[TrafficEvent] = True
return distance
tree_one = real_tree()
tree_two = real_tree(path="data/c01-007-102/2/1129-2-process.csv")
prototype_names = ["1", "2"]
prototype_caches = []
for index, tree in enumerate([tree_one, tree_two]):
prototype_caches.append(PrototypeSignatureCache.from_signature_caches(
[tree.to_index(signature=ParentChildByNameTopologySignature(),
supported={
ProcessStartEvent: True,
ProcessExitEvent: True,
TrafficEvent: True
},
statistics_cls=SetStatistics)],
prototype=prototype_names[index], threshold=0))
decorator = AnomalyDecorator()
algorithm = IncrementalDistanceAlgorithm(
signature=ParentChildByNameTopologySignature(),
distance=distance_builder,
cache_statistics=SetStatistics
)
prototype_cache = prototype_caches[0]
for cache in prototype_caches[1:]:
prototype_cache += cache
algorithm.cluster_representatives(
signature_prototypes=[prototype_cache],
prototypes=prototype_names
)
decorator.wrap_algorithm(algorithm)
for tree in [tree_one, tree_two]:
algorithm.start_tree()
for event in tree.event_iter(supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
# First tree vs. first is False for start
self.assertFalse(decorator.data()[0][0][0][0])
# ... and end
self.assertFalse(decorator.data()[0][0][0][-1])
# First tree vs. second is False for start
self.assertFalse(decorator.data()[0][0][1][0])
# and True for end
self.assertTrue(decorator.data()[0][0][1][-1])
# Second tree vs. first is False for start
self.assertFalse(decorator.data()[1][0][0][0])
# ... and True for end
self.assertTrue(decorator.data()[1][0][0][-1])
# Second tree vs. second is False for start
self.assertFalse(decorator.data()[1][0][1][0])
# ... and False for end
self.assertFalse(decorator.data()[1][0][1][-1])
def test_ensemble_result(self):
"""
I recognised that apparently sometimes (or ever?) values of ensemble
methods don't match results from single calculations. Therefore this
test should uniquely test for this.
"""
tree = real_tree()
tree_generator = TEDGenerator(
operation_generator=RandomOperation(
insert_probability=.5, delete_probability=.5),
costs=[TreeEditDistanceCost()],
seed=1234)
disturbed_tree = tree_generator.generate(tree)
signatures = [ParentChildByNameTopologySignature(),
ParentCountedChildrenByNameTopologySignature(count=2)]
# First test results from ensemble
ensemble_signature = EnsembleSignature(signatures=signatures)
decorator = DistanceMatrixDecorator(normalized=False)
algorithm = IncrementalDistanceAlgorithm(
signature=ensemble_signature,
distance=StartDistance
)
decorator.wrap_algorithm(algorithm)
algorithm.prototypes = [tree]
algorithm.start_tree()
for event in disturbed_tree.event_iter(supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
ensemble_data = decorator.data()
# Second, validate this result with single measurements
single_data = {}
for index, signature in enumerate(signatures):
decorator = DistanceMatrixDecorator(normalized=False)
algorithm = IncrementalDistanceAlgorithm(
signature=signature,
distance=StartDistance
)
decorator.wrap_algorithm(algorithm)
algorithm.prototypes = [real_tree()]
algorithm.start_tree()
for event in disturbed_tree.event_iter(supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
single_data[index] = decorator.data()
for index, _ in enumerate(signatures):
self.assertEqual(ensemble_data[0][index][0], single_data[index][0][0][0])
def test_ensemble_signature(self):
decorator = SignatureDecorator()
algorithm = IncrementalDistanceAlgorithm(signature=EnsembleSignature(
signatures=[
ParentChildByNameTopologySignature(),
ParentChildOrderByNameTopologySignature()
]))
algorithm.prototypes = [simple_prototype(), simple_monitoring_tree()]
decorator.wrap_algorithm(algorithm)
algorithm.start_tree()
for event in Event.from_tree(simple_monitoring_tree(),
supported={ProcessStartEvent: True}):
algorithm.add_event(event)
algorithm.finish_tree()
self.assertEqual(
decorator.descriptive_data(), {
'signature': [[[
'root_1', 'test_149160533', 'test_149160533',
'muh_149160533'
],
[
'.0_root_1', '.0.0_test_245236498',
'.0.0_test_245236498', '.0.1_muh_245236498'
]]]
})
algorithm.start_tree()
for event in Event.from_tree(simple_prototype(),
supported={ProcessStartEvent: True}):
algorithm.add_event(event)
algorithm.finish_tree()
self.assertEqual(
decorator.descriptive_data(), {
'signature':
[[[
'root_1', 'test_149160533', 'test_149160533',
'muh_149160533'
],
[
'.0_root_1', '.0.0_test_245236498',
'.0.0_test_245236498', '.0.1_muh_245236498'
]],
[[
'root_1', 'test_149160533', 'muh_149160533',
'test_149160533', 'muh_149160533'
],
[
'.0_root_1', '.0.0_test_245236498', '.0.1_muh_245236498',
'.0.2_test_245236498', '.0.3_muh_245236498'
]]]
})
def test_two_prototypes(self):
decorator = DistanceDecorator(normalized=False)
algorithm = IncrementalDistanceAlgorithm()
algorithm.prototypes = [simple_prototype(), simple_monitoring_tree()]
decorator.wrap_algorithm(algorithm)
algorithm.start_tree()
self.assertEqual([[[[], []]]], decorator.data())
for event in Event.from_tree(simple_monitoring_tree(), supported={ProcessStartEvent: True}):
algorithm.add_event(event)
algorithm.finish_tree()
self.assertEqual([[[[2, 1, 1, 0], [2, 1, 1, 0]]]], decorator.data())
def test_empty_nodes(self):
signature = EnsembleSignature(
signatures=[ParentChildByNameTopologySignature(),
ParentCountedChildrenByNameTopologySignature(count=3)])
algorithm = IncrementalDistanceAlgorithm(signature=signature)
algorithm.prototypes = [simple_prototype()]
algorithm.start_tree()
for event in simple_monitoring_tree().event_iter(supported=algorithm.supported):
try:
distance = algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
self.assertEqual([[0], [7]], distance[-1])
def test_count_signature_for_correct_zero_distance(self):
signature = ParentCountedChildrenByNameTopologySignature(count=3)
algorithm = IncrementalDistanceAlgorithm(signature=signature)
decorator = DistanceMatrixDecorator(normalized=False)
decorator.wrap_algorithm(algorithm)
algorithm.prototypes = [real_tree()]
algorithm.start_tree()
for event in real_tree().event_iter(include_marker=True, supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
print(algorithm._signature_prototypes._prototype_dict[0]._prototype_dict.keys())
self.assertEqual([[[0]]], decorator.data())
def test_ensemble(self):
decorator = DistanceDecorator(normalized=False)
algorithm = IncrementalDistanceAlgorithm(
signature=EnsembleSignature(
signatures=[ParentChildByNameTopologySignature(),
ParentChildOrderByNameTopologySignature()]))
algorithm.prototypes = [simple_prototype()]
decorator.wrap_algorithm(algorithm)
algorithm.start_tree()
self.assertEqual([[[[]], [[]]]], decorator.data())
for event in Event.from_tree(simple_monitoring_tree(), supported={ProcessStartEvent: True}):
algorithm.add_event(event)
algorithm.finish_tree()
self.assertEqual([[[[2, 1, 1, 0]], [[4, 3, 3, 2]]]], decorator.data())
def test_attribute_distance(self):
def distance_buidler(**kwargs):
distance = StartExitDistance(weight=0, **kwargs)
distance.supported = {
ProcessStartEvent: True,
ProcessExitEvent: True,
TrafficEvent: True
}
return distance
tree_builder = CSVTreeBuilder()
tree_1 = tree_builder.build(
os.path.join(os.path.dirname(assess_tests.__file__),
"data/c01-007-102/2/1129-2-process.csv"))
tree_2 = tree_builder.build(
os.path.join(os.path.dirname(assess_tests.__file__),
"data/c01-007-102/2/1136-3-process.csv"))
signature = ParentChildByNameTopologySignature()
algorithm = IncrementalDistanceAlgorithm(
signature=signature,
distance=distance_buidler,
cache_statistics=SplittedStatistics)
algorithm.prototypes = [tree_1, tree_2]
decorator = DistanceMatrixDecorator(normalized=False)
decorator.wrap_algorithm(algorithm)
algorithm.start_tree()
for event in tree_1.event_iter(supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
algorithm.start_tree()
for event in tree_2.event_iter(supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
data = decorator.data()
print(decorator.data())
self.assertEqual(4, abs(data[0][0][1] - data[1][0][0]))
def test_start_exit(self):
signature = ParentChildByNameTopologySignature()
alg = IncrementalDistanceAlgorithm(signature=signature,
distance=StartExitDistance)
alg.prototypes = [self._simple_prototype]
alg.start_tree()
alg.add_event(Event.start(tme=0, pid=2, ppid=1, name="root_node"))
alg.add_event(Event.start(tme=1, pid=3, ppid=2, name="first_child"))
alg.add_event(Event.start(tme=1, pid=4, ppid=2, name="second_child"))
alg.add_event(
Event.exit(tme=3, start_tme=1, pid=3, ppid=2, name="first_child"))
alg.add_event(
Event.exit(tme=4, start_tme=1, pid=4, ppid=2, name="second_child"))
alg.add_event(Event.start(tme=5, pid=5, ppid=2, name="first_child"))
alg.add_event(
Event.exit(tme=7, start_tme=5, pid=5, ppid=2, name="first_child"))
distance = alg.add_event(
Event.exit(tme=10, start_tme=0, pid=2, ppid=1, name="root_node"))
alg.finish_tree()
self.assertEqual(distance[0][0], [0])
def test_node_count_for_correct_zero_distance(self):
signature = EnsembleSignature(
signatures=[ParentChildByNameTopologySignature(),
ParentCountedChildrenByNameTopologySignature(count=3)])
algorithm = IncrementalDistanceAlgorithm(
signature=signature, distance=SimpleDistance)
data_decorator = DataDecorator()
data_decorator.wrap_algorithm(algorithm)
algorithm.prototypes = [real_tree()]
algorithm.start_tree()
for event in real_tree().event_iter(include_marker=True, supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
self.assertEqual([tree_value for values in data_decorator.data().get(
"prototypes", {}).get("converted", []) for tree_value in values],
[tree_value for values in data_decorator.data().get(
"monitoring", {}).get("converted", []) for tree_value in values])
def test_symmetry_optimisation(self):
tree = CSVTreeBuilder().build(
os.path.join(os.path.dirname(assess_tests.__file__),
"data/c01-007-102/2/1129-2-process.csv"))
signature = ParentChildByNameTopologySignature()
algorithm = IncrementalDistanceAlgorithm(
signature=signature,
distance=lambda **kwargs: StartExitDistance(weight=0, **kwargs),
cache_statistics=SplittedStatistics)
algorithm.prototypes = [tree]
decorator = DistanceMatrixDecorator(normalized=False)
decorator.wrap_algorithm(algorithm)
algorithm.start_tree()
for event in tree.event_iter(supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
self.assertEqual(0, decorator.data()[0][0][0])
def test_attributes_nodes_only(self):
def distance(**kwargs):
distance = StartExitDistance(weight=1, **kwargs)
distance.supported[TrafficEvent] = True
return distance
algorithm = IncrementalDistanceAlgorithm(
signature=ParentChildByNameTopologySignature(),
distance=distance,
cache_statistics=SetStatistics
)
decorator = DistanceMatrixDecorator(normalized=False)
decorator.wrap_algorithm(algorithm)
algorithm.prototypes = [real_tree("data/c01-007-102/2/1129-2-process.csv")]
monitoring_tree = real_tree("data/c01-007-102/2/1129-2-process.csv")
algorithm.start_tree()
for event in monitoring_tree.event_iter(supported=algorithm.supported):
try:
algorithm.add_event(event)
except EventNotSupportedException:
pass
algorithm.finish_tree()
self.assertEqual([[[0]]], decorator.data())
def test_initialisation_of_cluster_representatitives(self):
def distance_builder(**kwargs):
distance = StartExitDistance(weight=.5, **kwargs)
distance.supported[TrafficEvent] = True
return distance
tree_one = real_tree()
tree_two = real_tree(path="data/c01-007-102/2/1129-2-process.csv")
tree_three = real_tree(path="data/c01-007-102/2/1136-3-process.csv")
distance = StartExitDistance(weight=.5)
distance.supported[TrafficEvent] = True
signature = ParentChildByNameTopologySignature()
tree_profiles = [
tree_one.to_index(signature=signature,
supported=distance.supported,
statistics_cls=SetStatistics),
tree_two.to_index(signature=signature,
supported=distance.supported,
statistics_cls=SetStatistics),
tree_three.to_index(signature=signature,
supported=distance.supported,
statistics_cls=SetStatistics)
]
algorithm = IncrementalDistanceAlgorithm(
signature=ParentChildByNameTopologySignature(),
distance=distance_builder,
cache_statistics=SetStatistics)
cluster_distance = ClusterDistance(distance=distance)
prototype_names = ["test"]
prototype_signatures = []
for prototype in prototype_names:
prototype_signatures.append(
cluster_distance.mean(tree_profiles, prototype=prototype))
algorithm.cluster_representatives(
signature_prototypes=prototype_signatures,
prototypes=prototype_names)
algorithm.start_tree()
