def __init__(self, pattern_to_tree_plan_map: Dict[Pattern, TreePlan],
storage_params: TreeStorageParameters,
statistics_collector: StatisticsCollector = None,
optimizer: Optimizer = None,
statistics_update_time_window: timedelta = None):
self.__is_multi_pattern_mode = len(pattern_to_tree_plan_map) > 1
if self.__is_multi_pattern_mode:
# TODO: support statistic collection in the multi-pattern mode
self._tree = MultiPatternTree(pattern_to_tree_plan_map, storage_params)
else:
pattern = list(pattern_to_tree_plan_map)[0]
pattern.condition.set_statistics_collector(statistics_collector)
self._tree = Tree(list(pattern_to_tree_plan_map.values())[0],
list(pattern_to_tree_plan_map)[0], storage_params)
self.__storage_params = storage_params
self.__statistics_collector = statistics_collector
self.__optimizer = optimizer
self._event_types_listeners = {}
self.__statistics_update_time_window = statistics_update_time_window
# The remainder of the initialization process is only relevant for the freeze map feature. This feature can
# only be enabled in single-pattern mode.
self._pattern = list(pattern_to_tree_plan_map)[0] if not self.__is_multi_pattern_mode else None
self.__freeze_map = {}
self.__active_freezers = []
if not self.__is_multi_pattern_mode and self._pattern.consumption_policy is not None and \
self._pattern.consumption_policy.freeze_names is not None:
self.__init_freeze_map()
def test_imbalanced_tree(self):
root = Tree(6)
root.set_left(5)
root.left.set_left(2)
root.left.set_right(3)
answer = [['|', '|', '|', '6', '|', '|', '|']]
answer.append(['|', '5', '|', '|', '|', '|', '|'])
answer.append(['2', '|', '3', '|', '|', '|', '|'])
assert root.print_tree() == answer
def test_create_tree_2(self):
# A tree with root, three children.
t = Tree('r')
t._add(t.root(), 'a')
t._add(t.root(), 'b')
t._add(t.root(), 'c')
self.assertEqual(t.height(), 1)
self.assertEqual(len(t), 4)
def test_add_between(self):
t = Tree('r')
r = t.root()
a = t.add(r, 'a')
b = t.add(a, 'b')
c = t.add(a, 'c')
d = t.add(a, 'd')
e = t.add_between(r, a, 'e')
f = t.add(e, 'f')
self.assertEqual([x.element() for x in t.bfs()],
['r', 'e', 'a', 'f', 'b', 'c', 'd'])
self.assertEqual(len(t), 7)
self.assertEqual(t.height(), 3)
def test_level_order(self):
t = Tree('r')
r = t.root()
a = t.add(r, 'a')
b = t.add(a, 'b')
c = t.add(a, 'c')
d = t.add(a, 'd')
e = t.add_between(r, a, 'e')
f = t.add(e, 'f')
self.assertEqual([[x.element() for x in level]
for level in t.level_traversal()],
[['r'], ['e'], ['a', 'f'], ['b', 'c', 'd']])
def __construct_trees_for_patterns(self, pattern_to_tree_plan_map: Dict[Pattern, TreePlan],
storage_params: TreeStorageParameters):
"""
Creates a list of tree objects corresponding to the specified tree plans.
"""
i = 1 # pattern IDs starts from 1
trees = []
for pattern, plan in pattern_to_tree_plan_map.items():
trees.append(Tree(plan, pattern, storage_params, i))
i += 1
return trees
def __init__(self, pattern_to_tree_plan_map: Dict[Pattern, TreePlan],
storage_params: TreeStorageParameters,
multi_pattern_eval_params: MultiPatternEvaluationParameters = MultiPatternEvaluationParameters()):
is_multi_pattern_mode = len(pattern_to_tree_plan_map) > 1
if is_multi_pattern_mode:
self.__tree = MultiPatternTree(pattern_to_tree_plan_map, storage_params, multi_pattern_eval_params)
else:
self.__tree = Tree(list(pattern_to_tree_plan_map.values())[0],
list(pattern_to_tree_plan_map)[0], storage_params)
self.__event_types_listeners = {}
# The remainder of the initialization process is only relevant for the freeze map feature. This feature can
# only be enabled in single-pattern mode.
self.__pattern = list(pattern_to_tree_plan_map)[0] if not is_multi_pattern_mode else None
self.__freeze_map = {}
self.__active_freezers = []
if not is_multi_pattern_mode and self.__pattern.consumption_policy is not None and \
self.__pattern.consumption_policy.freeze_names is not None:
self.__init_freeze_map()
def _register_event_listeners(tree: Tree):
"""
Given tree, register leaf listeners for event types.
"""
event_types_listeners = {}
for leaf in tree.get_leaves():
event_type = leaf.get_event_type()
if event_type in event_types_listeners.keys():
event_types_listeners[event_type].append(leaf)
else:
event_types_listeners[event_type] = [leaf]
return event_types_listeners
def genericSearch(self, root):
treeNode = TreeArrayListNode(root)
tree = Tree(treeNode)
vertexSet = set() #nodi da esaminare
markedNodes = set() #nodi gia' marcati per essere esaminati
markedNodes.add(root.index)
vertexSet.add(treeNode)
while len(vertexSet) > 0: #finche' ci sono nodi da esaminare
treeNode = vertexSet.pop() #un generico nodo non esaminato
nodes = self.foundNodesBySource(treeNode.info.index)
for nodeIndex in nodes:
if nodeIndex not in markedNodes: #crea il nodo per l'albero e
#collega padre e figlio
newTreeNode = TreeArrayListNode(self.nodes[nodeIndex])
markedNodes.add(nodeIndex)
newTreeNode.father = treeNode
treeNode.sons.append(newTreeNode)
vertexSet.add(newTreeNode)
else:
currNode = tree.foundNodeByIndex(nodeIndex)
#TODO: aggiorna il padre
return tree
def test_balanced_tree(self):
root = Tree(1)
root.set_left(4)
root.set_right(5)
root.left.set_left(3)
root.left.set_right(5)
root.right.set_left(7)
root.right.set_right(9)
answer = [['|', '|', '|', '1', '|', '|', '|']]
answer.append(['|', '4', '|', '|', '|', '5', '|'])
answer.append(['3', '|', '5', '|', '7', '|', '9'])
assert root.print_tree() == answer
def __perform_reoptimization(self, last_statistics_refresh_time: timedelta, last_event: Event):
"""
If needed, reoptimizes the evaluation mechanism to reflect the current statistical properties of the
input event stream.
"""
self.__statistics_collector.handle_event(last_event)
if not self._should_try_reoptimize(last_statistics_refresh_time, last_event):
# it is not yet time to recalculate the statistics
return last_statistics_refresh_time
new_statistics = self.__statistics_collector.get_statistics()
if self.__optimizer.should_optimize(new_statistics, self._pattern):
new_tree_plan = self.__optimizer.build_new_plan(new_statistics, self._pattern)
new_tree = Tree(new_tree_plan, self._pattern, self.__storage_params)
self._tree_update(new_tree, last_event.timestamp)
# this is the new last statistic refresh time
return last_event.timestamp
def __construct_multi_pattern_tree(
self, pattern_to_tree_plan_map: Dict[Pattern, TreePlan],
storage_params: TreeStorageParameters):
"""
Constructs a multi-pattern evaluation tree.
It is assumed that each pattern appears only once in patterns (which is a legitimate assumption).
"""
i = 1 # pattern IDs starts from 1
plan_nodes_to_nodes_map = {} # a cache for already created subtrees
for pattern, plan in pattern_to_tree_plan_map.items():
pattern.id = i
new_tree_root = Tree(plan, pattern, storage_params,
plan_nodes_to_nodes_map).get_root()
self.__id_to_output_node_map[pattern.id] = new_tree_root
self.__id_to_pattern_map[pattern.id] = pattern
self.__output_nodes.append(new_tree_root)
i += 1
from tree.Tree import Tree
__author__ = 'lijiayan'
if __name__ == '__main__':
tree = Tree()
tree.add(1)
tree.add(2)
tree.add(3)
tree.add(4)
tree.add(5)
tree.add(6)
tree.breadth_travel()
print("===========")
tree.preOrder(tree.root)
from tree.Tree import Tree from In_out.Peripheric_manager import Peripheric_manager from data_manager.utils.Getter import Getter from data_manager.read_tree.configure_peripherics import config_peripherics from data_manager.read_tree.configure_tree import config_tree from data_manager.read_tree.reload_tree import reload_tree from In_out.network.Server import Server """ Create the tree and Peripheric_manager """ tree = Tree() manager = Peripheric_manager() getter = Getter(tree, manager) config_peripherics(getter) config_tree(getter) Server(getter).start()
def test_create_tree_1(self):
# A tree with only a root.
t = Tree()
self.assertEqual(len(t), 1)
def test_one_node_tree(self):
assert Tree(2).print_tree() == [['2']]
def test_create_tree_0(self):
t = Tree(None)
self.assertEqual(t.root().element(), None)
def test_bfs_1(self):
t = Tree('a')
b = t._add(t.root(), 'b')
t._add(b, 'd')
c = t._add(t.root(), 'c')
t._add(c, 'e')
f = t._add(c, 'f')
t._add(f, 'g')
self.assertEqual([x.element() for x in t.bfs()],
['a', 'b', 'c', 'd', 'e', 'f', 'g'])
def test_create_tree_3(self):
t = Tree()
a = t._add(t.root(), 'a')
t._add(t.root(), 'b')
t._add(t.root(), 'c')
t._add(a, 'aa')
self.assertEqual(t.height(), 2)
# Single child node of node a contains value 'aa' as element.
self.assertEqual(t.children(a)[0].element(), 'aa')
def test_get_height_1(self):
root = Tree(2)
assert root.get_height() == 1
class TreeBasedEvaluationMechanism(EvaluationMechanism, ABC):
"""
An implementation of the tree-based evaluation mechanism.
"""
def __init__(self, pattern_to_tree_plan_map: Dict[Pattern, TreePlan],
storage_params: TreeStorageParameters,
statistics_collector: StatisticsCollector = None,
optimizer: Optimizer = None,
statistics_update_time_window: timedelta = None):
self.__is_multi_pattern_mode = len(pattern_to_tree_plan_map) > 1
if self.__is_multi_pattern_mode:
# TODO: support statistic collection in the multi-pattern mode
self._tree = MultiPatternTree(pattern_to_tree_plan_map, storage_params)
else:
pattern = list(pattern_to_tree_plan_map)[0]
pattern.condition.set_statistics_collector(statistics_collector)
self._tree = Tree(list(pattern_to_tree_plan_map.values())[0],
list(pattern_to_tree_plan_map)[0], storage_params)
self.__storage_params = storage_params
self.__statistics_collector = statistics_collector
self.__optimizer = optimizer
self._event_types_listeners = {}
self.__statistics_update_time_window = statistics_update_time_window
# The remainder of the initialization process is only relevant for the freeze map feature. This feature can
# only be enabled in single-pattern mode.
self._pattern = list(pattern_to_tree_plan_map)[0] if not self.__is_multi_pattern_mode else None
self.__freeze_map = {}
self.__active_freezers = []
if not self.__is_multi_pattern_mode and self._pattern.consumption_policy is not None and \
self._pattern.consumption_policy.freeze_names is not None:
self.__init_freeze_map()
def eval(self, events: InputStream, matches: OutputStream, data_formatter: DataFormatter):
"""
Activates the tree evaluation mechanism on the input event stream and reports all found pattern matches to the
given output stream.
"""
self._event_types_listeners = self._register_event_listeners(self._tree)
last_statistics_refresh_time = None
for raw_event in events:
event = Event(raw_event, data_formatter)
if event.type not in self._event_types_listeners.keys():
continue
self.__remove_expired_freezers(event)
if not self.__is_multi_pattern_mode and self.__statistics_collector is not None:
# TODO: support multi-pattern mode
last_statistics_refresh_time = self.__perform_reoptimization(last_statistics_refresh_time, event)
self._play_new_event_on_tree(event, matches)
self._get_matches(matches)
# Now that we finished the input stream, if there were some pending matches somewhere in the tree, we will
# collect them now
self._get_last_pending_matches(matches)
matches.close()
def __perform_reoptimization(self, last_statistics_refresh_time: timedelta, last_event: Event):
"""
If needed, reoptimizes the evaluation mechanism to reflect the current statistical properties of the
input event stream.
"""
self.__statistics_collector.handle_event(last_event)
if not self._should_try_reoptimize(last_statistics_refresh_time, last_event):
# it is not yet time to recalculate the statistics
return last_statistics_refresh_time
new_statistics = self.__statistics_collector.get_statistics()
if self.__optimizer.should_optimize(new_statistics, self._pattern):
new_tree_plan = self.__optimizer.build_new_plan(new_statistics, self._pattern)
new_tree = Tree(new_tree_plan, self._pattern, self.__storage_params)
self._tree_update(new_tree, last_event.timestamp)
# this is the new last statistic refresh time
return last_event.timestamp
def _should_try_reoptimize(self, last_statistics_refresh_time: timedelta, last_event: Event):
"""
Returns True if statistic recalculation and a reoptimization attempt can now be performed and False otherwise.
The default implementation merely checks whether enough time has passed since the last reoptimization attempt.
"""
if last_statistics_refresh_time is None:
return True
return last_event.timestamp - last_statistics_refresh_time > self.__statistics_update_time_window
def _get_last_pending_matches(self, matches):
"""
Collects the pending matches from the tree
"""
for match in self._tree.get_last_matches():
matches.add_item(match)
def _play_new_event(self, event: Event, event_types_listeners):
"""
Lets the tree handle the event
"""
for leaf in event_types_listeners[event.type]:
if self._should_ignore_events_on_leaf(leaf, event_types_listeners):
continue
self.__try_register_freezer(event, leaf)
leaf.handle_event(event)
def _get_matches(self, matches: OutputStream):
"""
Collects the ready matches from the tree and adds them to the evaluation matches.
"""
for match in self._tree.get_matches():
matches.add_item(match)
self._remove_matched_freezers(match.events)
@staticmethod
def _register_event_listeners(tree: Tree):
"""
Given tree, register leaf listeners for event types.
"""
event_types_listeners = {}
for leaf in tree.get_leaves():
event_type = leaf.get_event_type()
if event_type in event_types_listeners.keys():
event_types_listeners[event_type].append(leaf)
else:
event_types_listeners[event_type] = [leaf]
return event_types_listeners
def __init_freeze_map(self):
"""
For each event type specified by the user to be a 'freezer', that is, an event type whose appearance blocks
initialization of new sequences until it is either matched or expires, this method calculates the list of
leaves to be disabled.
"""
sequences = self._pattern.extract_flat_sequences()
for freezer_event_name in self._pattern.consumption_policy.freeze_names:
current_event_name_set = set()
for sequence in sequences:
if freezer_event_name not in sequence:
continue
for name in sequence:
current_event_name_set.add(name)
if name == freezer_event_name:
break
if len(current_event_name_set) > 0:
self.__freeze_map[freezer_event_name] = current_event_name_set
def _should_ignore_events_on_leaf(self, leaf: LeafNode, event_types_listeners):
"""
If the 'freeze' consumption policy is enabled, checks whether the given event should be dropped based on it.
"""
if len(self.__freeze_map) == 0:
# freeze option disabled
return False
for freezer in self.__active_freezers:
for freezer_leaf in event_types_listeners[freezer.type]:
if freezer_leaf.get_event_name() not in self.__freeze_map:
continue
if leaf.get_event_name() in self.__freeze_map[freezer_leaf.get_event_name()]:
return True
return False
def __try_register_freezer(self, event: Event, leaf: LeafNode):
"""
Check whether the current event is a freezer event, and, if positive, register it.
"""
if leaf.get_event_name() in self.__freeze_map.keys():
self.__active_freezers.append(event)
def _remove_matched_freezers(self, match_events: List[Event]):
"""
Removes the freezers that have been matched.
"""
if len(self.__freeze_map) == 0:
# freeze option disabled
return False
self.__active_freezers = [freezer for freezer in self.__active_freezers if freezer not in match_events]
def __remove_expired_freezers(self, event: Event):
"""
Removes the freezers that have been expired.
"""
if len(self.__freeze_map) == 0:
# freeze option disabled
return False
self.__active_freezers = [freezer for freezer in self.__active_freezers
if event.timestamp - freezer.timestamp <= self._pattern.window]
def get_structure_summary(self):
return self._tree.get_structure_summary()
def __repr__(self):
return self.get_structure_summary()
def _tree_update(self, new_tree: Tree, event: Event):
"""
Registers a new tree in the evaluation mechanism.
"""
raise NotImplementedError()
def _play_new_event_on_tree(self, event: Event, matches: OutputStream):
"""
Lets the tree handle the event.
"""
raise NotImplementedError()
def test_get_height_2(self):
root = Tree(2)
root.set_left(3)
assert root.get_height() == 2
class TreeBasedEvaluationMechanism(EvaluationMechanism):
"""
An implementation of the tree-based evaluation mechanism.
"""
def __init__(self, pattern_to_tree_plan_map: Dict[Pattern, TreePlan],
storage_params: TreeStorageParameters,
multi_pattern_eval_params: MultiPatternEvaluationParameters = MultiPatternEvaluationParameters()):
is_multi_pattern_mode = len(pattern_to_tree_plan_map) > 1
if is_multi_pattern_mode:
self.__tree = MultiPatternTree(pattern_to_tree_plan_map, storage_params, multi_pattern_eval_params)
else:
self.__tree = Tree(list(pattern_to_tree_plan_map.values())[0],
list(pattern_to_tree_plan_map)[0], storage_params)
self.__event_types_listeners = {}
# The remainder of the initialization process is only relevant for the freeze map feature. This feature can
# only be enabled in single-pattern mode.
self.__pattern = list(pattern_to_tree_plan_map)[0] if not is_multi_pattern_mode else None
self.__freeze_map = {}
self.__active_freezers = []
if not is_multi_pattern_mode and self.__pattern.consumption_policy is not None and \
self.__pattern.consumption_policy.freeze_names is not None:
self.__init_freeze_map()
def eval(self, events: InputStream, matches: OutputStream, data_formatter: DataFormatter):
"""
Activates the tree evaluation mechanism on the input event stream and reports all found pattern matches to the
given output stream.
"""
self.__register_event_listeners()
for raw_event in events:
event = Event(raw_event, data_formatter)
if event.type not in self.__event_types_listeners.keys():
continue
self.__remove_expired_freezers(event)
for leaf in self.__event_types_listeners[event.type]:
if self.__should_ignore_events_on_leaf(leaf):
continue
self.__try_register_freezer(event, leaf)
leaf.handle_event(event)
for match in self.__tree.get_matches():
matches.add_item(match)
self.__remove_matched_freezers(match.events)
# Now that we finished the input stream, if there were some pending matches somewhere in the tree, we will
# collect them now
for match in self.__tree.get_last_matches():
matches.add_item(match)
matches.close()
def __register_event_listeners(self):
"""
Register leaf listeners for event types.
"""
self.__event_types_listeners = {}
for leaf in self.__tree.get_leaves():
event_type = leaf.get_event_type()
if event_type in self.__event_types_listeners.keys():
self.__event_types_listeners[event_type].append(leaf)
else:
self.__event_types_listeners[event_type] = [leaf]
def __init_freeze_map(self):
"""
For each event type specified by the user to be a 'freezer', that is, an event type whose appearance blocks
initialization of new sequences until it is either matched or expires, this method calculates the list of
leaves to be disabled.
"""
sequences = self.__pattern.extract_flat_sequences()
for freezer_event_name in self.__pattern.consumption_policy.freeze_names:
current_event_name_set = set()
for sequence in sequences:
if freezer_event_name not in sequence:
continue
for name in sequence:
current_event_name_set.add(name)
if name == freezer_event_name:
break
if len(current_event_name_set) > 0:
self.__freeze_map[freezer_event_name] = current_event_name_set
def __should_ignore_events_on_leaf(self, leaf: LeafNode):
"""
If the 'freeze' consumption policy is enabled, checks whether the given event should be dropped based on it.
"""
if len(self.__freeze_map) == 0:
# freeze option disabled
return False
for freezer in self.__active_freezers:
for freezer_leaf in self.__event_types_listeners[freezer.type]:
if freezer_leaf.get_event_name() not in self.__freeze_map:
continue
if leaf.get_event_name() in self.__freeze_map[freezer_leaf.get_event_name()]:
return True
return False
def __try_register_freezer(self, event: Event, leaf: LeafNode):
"""
Check whether the current event is a freezer event, and, if positive, register it.
"""
if leaf.get_event_name() in self.__freeze_map.keys():
self.__active_freezers.append(event)
def __remove_matched_freezers(self, match_events: List[Event]):
"""
Removes the freezers that have been matched.
"""
if len(self.__freeze_map) == 0:
# freeze option disabled
return False
self.__active_freezers = [freezer for freezer in self.__active_freezers if freezer not in match_events]
def __remove_expired_freezers(self, event: Event):
"""
Removes the freezers that have been expired.
"""
if len(self.__freeze_map) == 0:
# freeze option disabled
return False
self.__active_freezers = [freezer for freezer in self.__active_freezers
if event.timestamp - freezer.timestamp <= self.__pattern.window]
def get_structure_summary(self):
return self.__tree.get_structure_summary()
def __repr__(self):
return self.get_structure_summary()