def test_1_level_10_increments_1_event_per_increment(self):
buffer = SharedVersionedMatchBuffer()
run_id = generate_unique_string()
events = [
PrimitiveEvent(timestamp=EpochNSClock.generate_timestamp())
for _ in range(10)
]
version = RunVersion()
version.add_level(run_id)
version_current = version.get_version_as_str()
for event in events:
version.increment_level(generate_unique_string())
version_next = version.get_version_as_str()
buffer.put_event(nfa_name=NFA_NAME_A,
run_id=run_id,
version=version_current,
state_label=LABEL_LAYER_A,
event=event,
new_version=version_next)
version_current = version_next
history_events = buffer.get_all_events(
nfa_name=NFA_NAME_A, run_id=run_id,
version=version).events[LABEL_LAYER_A]
self.assertEqual(10, len(history_events))
for event in events:
self.assertTrue(event in history_events)
def test_2_nfas_2_versions_1_increment(self):
buffer = SharedVersionedMatchBuffer()
run_id_a = generate_unique_string()
run_id_a_incr = generate_unique_string()
run_id_b = generate_unique_string()
# create two versions for run a and run b
version_a = RunVersion()
version_a.add_level(run_id_a)
version_a_str = version_a.get_version_as_str()
version_b = RunVersion()
version_b.add_level(run_id_b)
# add event a into version a, run a
buffer.put_event(nfa_name=NFA_NAME_A,
run_id=run_id_a,
version=version_a_str,
state_label=LABEL_LAYER_A,
event=event_a)
# increment version a
version_a.increment_level(generate_unique_string())
version_a_incr_str = version_a.get_version_as_str()
# add event b to version a incr, run a incr
buffer.put_event(nfa_name=NFA_NAME_A,
run_id=run_id_a,
version=version_a_str,
state_label=LABEL_LAYER_B,
event=event_b,
new_run_id=run_id_a_incr,
new_version=version_a_incr_str)
# add event c to version b, run b
buffer.put_event(nfa_name=NFA_NAME_B,
run_id=run_id_b,
version=version_b.get_version_as_str(),
state_label=LABEL_LAYER_C,
event=event_c)
# should only return events associated with version a and incr
history_1 = buffer.get_all_events(nfa_name=NFA_NAME_A,
run_id=run_id_a_incr,
version=version_a)
self.assertDictEqual(history_1.events, {
LABEL_LAYER_A: [event_a],
LABEL_LAYER_B: [event_b]
})
# should only return events associated with version 2 and run 2
history_2 = buffer.get_all_events(nfa_name=NFA_NAME_B,
run_id=run_id_b,
version=version_b)
self.assertDictEqual(history_2.events, {LABEL_LAYER_C: [event_c]})
def test_remove_version_1_nfa_2_versions_1_event(self):
buffer = SharedVersionedMatchBuffer()
# two runs for the same nfa
run_id_a = generate_unique_string()
run_id_b = generate_unique_string()
# version for each run
version_a = RunVersion()
version_a.add_level(run_id_a)
version_a_str = version_a.get_version_as_str()
version_b = RunVersion()
version_b.add_level(run_id_b)
version_b_str = version_b.get_version_as_str()
# put event for run a
buffer.put_event(nfa_name=NFA_NAME_A,
run_id=run_id_a,
version=version_a_str,
state_label=LABEL_LAYER_B,
event=event_a)
# match event for event a, run a
match_event = buffer._eve[NFA_NAME_A][LABEL_LAYER_B][event_a.event_id]
self.assertIsNotNone(match_event)
self.assertEqual(event_a, match_event.event)
self.assertTrue(version_a_str in match_event.next_ids)
# put event for run b
buffer.put_event(nfa_name=NFA_NAME_A,
run_id=run_id_b,
version=version_b_str,
state_label=LABEL_LAYER_B,
event=event_a)
# run b in same match event
self.assertTrue(version_b_str in match_event.next_ids)
# remove version a, match event should remain for version b
buffer.remove_version(nfa_name=NFA_NAME_A, version=version_a_str)
match_event = buffer._eve[NFA_NAME_A][LABEL_LAYER_B][event_a.event_id]
self.assertIsNotNone(match_event)
self.assertFalse(version_a_str in match_event.next_ids)
self.assertTrue(version_b_str in match_event.next_ids)
# remove version b, match event should be removed from the buffer
buffer.remove_version(nfa_name=NFA_NAME_A, version=version_b_str)
with self.assertRaises(KeyError):
match_event = buffer._eve[NFA_NAME_A][LABEL_LAYER_B][
event_a.event_id]
self.assertFalse(version_b_str in match_event.next_ids)
def test_sliding_first_window_to_halt(self):
predicate_a = BoboPredicateCallable(predicate_key_a_value_a)
predicate_b = BoboPredicateCallable(predicate_key_a_value_b)
predicate_c = BoboPredicateCallable(predicate_key_a_value_c)
predicate_first_window = WindowSlidingFirst(
interval_sec=range_sec)
event_a = PrimitiveEvent(timestamp_low, {KEY: VAL_1})
event_b = PrimitiveEvent(timestamp_mid, {KEY: VAL_2})
event_c = PrimitiveEvent(timestamp_upp, {KEY: VAL_3})
pattern_a = BoboPattern() \
.followed_by(LABEL_LAYER_A, predicate_a) \
.followed_by(LABEL_LAYER_B, predicate_b) \
.followed_by(LABEL_LAYER_C, predicate_c) \
.precondition(predicate_first_window)
handler = BoboNFAHandler(
BoboRuleBuilder.nfa(NFA_NAME_A, pattern_a),
SharedVersionedMatchBuffer())
handlersub = NFAHandlerSubscriber()
handler.subscribe(handlersub)
handler.process(event_a)
handler.process(event_b)
handler.process(event_c)
self.assertEqual(len(handlersub.final), 1)
def test_bobo_run(self):
buffer_orig = SharedVersionedMatchBuffer()
nfa_orig = BoboRuleBuilder.nfa(name_nfa=NFA_NAME_A,
pattern=stub_pattern)
original = BoboRun(buffer=buffer_orig, nfa=nfa_orig, event=event_a)
generated = BoboDeciderBuilder \
.run(d=original.to_dict(),
buffer=original.buffer,
nfa=original.nfa)
self.assertEqual(original.buffer, generated.buffer)
self.assertEqual(original.nfa, generated.nfa)
self.assertDictEqual(original.event.to_dict(),
generated.event.to_dict())
self.assertEqual(original.start_time, generated.start_time)
self.assertEqual(original.start_state, generated.start_state)
self.assertEqual(original.current_state, generated.current_state)
self.assertEqual(original.id, generated.id)
self.assertEqual(original.version.get_version_as_str(),
generated.version.get_version_as_str())
self.assertEqual(original.last_process_cloned(),
generated.last_process_cloned())
self.assertEqual(original.is_halted(), generated.is_halted())
def test_sliding_last_window_to_halt(self):
timestamp_lower = EpochNSClock.generate_timestamp()
sleep(0.5)
timestamp_upper = EpochNSClock.generate_timestamp()
window_range_ns = timestamp_upper - timestamp_lower
timestamp_a = timestamp_lower
timestamp_b = timestamp_a + window_range_ns
timestamp_c = timestamp_b + window_range_ns + 1
predicate_a = BoboPredicateCallable(predicate_key_a_value_a)
predicate_b = BoboPredicateCallable(predicate_key_a_value_b)
predicate_c = BoboPredicateCallable(predicate_key_a_value_c)
predicate_first_window = WindowSlidingLast(window_range_ns)
event_a = PrimitiveEvent(timestamp_a, {KEY: VAL_1})
event_b = PrimitiveEvent(timestamp_b, {KEY: VAL_2})
event_c = PrimitiveEvent(timestamp_c, {KEY: VAL_3})
pattern_a = BoboPattern() \
.followed_by(LABEL_LAYER_A, predicate_a) \
.followed_by(LABEL_LAYER_B, predicate_b) \
.followed_by(LABEL_LAYER_C, predicate_c) \
.precondition(predicate_first_window)
handler = BoboNFAHandler(BoboRuleBuilder.nfa(NFA_NAME_A, pattern_a),
SharedVersionedMatchBuffer())
handlersub = NFAHandlerSubscriber()
handler.subscribe(handlersub)
handler.process(event_a)
handler.process(event_b)
handler.process(event_c)
self.assertEqual(len(handlersub.final), 1)
def _config_definitions(self) -> None:
for event_def in self._event_defs:
handler = BoboNFAHandler(nfa=BoboRuleBuilder.nfa(
name_nfa=event_def.name, pattern=event_def.pattern),
buffer=SharedVersionedMatchBuffer(),
max_recent=self._max_recent)
self._decider.add_nfa_handler(handler)
# Decider -> Producer
self._decider.subscribe(event_def.name, self._producer)
if self._recursive:
# Producer -> Decider
self._producer.subscribe(event_def.name, self._decider)
# Producer -> Forwarder
self._producer.subscribe(event_def.name, self._forwarder)
if event_def.action is not None:
# Producer -> Action
self._producer.subscribe(event_def.name, event_def.action)
# Action -> Producer
event_def.action.subscribe(self._producer)
if self._distributed:
# Handler -> Outgoing
handler.subscribe(self._manager.outgoing)
# Producer -> Outgoing
self._producer.subscribe(event_def.name,
self._manager.outgoing)
def handler_setup(nfa_name, pattern, max_recent: int = 1):
buffer = SharedVersionedMatchBuffer()
nfa = BoboRuleBuilder.nfa(name_nfa=nfa_name, pattern=pattern)
handler = BoboNFAHandler(nfa=nfa, buffer=buffer, max_recent=max_recent)
handlersub = NFAHandlerSubscriber()
handler.subscribe(handlersub)
return nfa, buffer, handler, handlersub
def test_shared_versioned_match_buffer(self):
original = SharedVersionedMatchBuffer()
original.put_event(nfa_name=NFA_NAME_A,
run_id=RUN_ID_A,
version=VERSION_A,
state_label=LABEL_LAYER_A,
event=event_a)
# ensure event is found in original buffer
event_orig = original.get_event(nfa_name=NFA_NAME_A,
state_label=LABEL_LAYER_A,
event_id=event_a.event_id,
default=None)
self.assertIsNotNone(event_orig)
self.assertEqual(event_orig, event_a)
# generate new buffer
generated = BoboDeciderBuilder \
.shared_versioned_match_buffer(original.to_dict())
# ensure event is found in generated buffer
event_gen = generated.get_event(nfa_name=NFA_NAME_A,
state_label=LABEL_LAYER_A,
event_id=event_a.event_id,
default=None)
self.assertIsNotNone(event_gen)
# ensure events match between buffers
self.assertDictEqual(event_orig.to_dict(), event_gen.to_dict())
def shared_versioned_match_buffer(d: dict) -> SharedVersionedMatchBuffer:
"""
:param d: A dict representation of a SharedVersionedMatchBuffer
instance.
:type d: dict
:return: A new SharedVersionedMatchBuffer instance.
"""
buffer = SharedVersionedMatchBuffer()
for eve_dict in d[SharedVersionedMatchBuffer.EVENTS]:
nfa_name = eve_dict[SharedVersionedMatchBuffer.NFA_NAME]
match_event = BoboDeciderBuilder.match_event(
eve_dict[SharedVersionedMatchBuffer.MATCH_EVENT])
nfa_labels = SharedVersionedMatchBuffer._get_or_create_subdict(
buffer._eve, nfa_name)
nfa_events = SharedVersionedMatchBuffer._get_or_create_subdict(
nfa_labels, match_event.label)
nfa_events[match_event.event.event_id] = match_event
for ver_dict in d[SharedVersionedMatchBuffer.LAST]:
nfa_name = ver_dict[SharedVersionedMatchBuffer.NFA_NAME]
label = ver_dict[SharedVersionedMatchBuffer.LABEL]
run_id = ver_dict[SharedVersionedMatchBuffer.RUN_ID]
version = ver_dict[SharedVersionedMatchBuffer.VERSION]
event_id = ver_dict[SharedVersionedMatchBuffer.EVENT_ID]
match_event = buffer._eve[nfa_name][label][event_id]
nfa_runs = SharedVersionedMatchBuffer._get_or_create_subdict(
buffer._ver, nfa_name)
run_versions = SharedVersionedMatchBuffer._get_or_create_subdict(
nfa_runs, run_id)
run_versions[version] = match_event
return buffer
def test_fixed_window_to_final(self):
timestamp_lower = EpochNSClock.generate_timestamp()
sleep(0.1)
timestamp_a = EpochNSClock.generate_timestamp()
sleep(0.1)
timestamp_b = EpochNSClock.generate_timestamp()
sleep(0.1)
timestamp_c = EpochNSClock.generate_timestamp()
sleep(0.1)
timestamp_upper = EpochNSClock.generate_timestamp()
sleep(0.1)
predicate_a = BoboPredicateCallable(predicate_key_a_value_a)
predicate_b = BoboPredicateCallable(predicate_key_a_value_b)
predicate_c = BoboPredicateCallable(predicate_key_a_value_c)
predicate_fixed_window = WindowFixed(timestamp_lower,
timestamp_upper)
event_a = PrimitiveEvent(timestamp_a, {KEY: VAL_1})
event_b = PrimitiveEvent(timestamp_b, {KEY: VAL_2})
event_c = PrimitiveEvent(timestamp_c, {KEY: VAL_3})
pattern_a = BoboPattern() \
.followed_by(LABEL_LAYER_A, predicate_a) \
.followed_by(LABEL_LAYER_B, predicate_b) \
.followed_by(LABEL_LAYER_C, predicate_c) \
.precondition(predicate_fixed_window)
handler = BoboNFAHandler(
BoboRuleBuilder.nfa(NFA_NAME_A, pattern_a),
SharedVersionedMatchBuffer())
handlersub = NFAHandlerSubscriber()
handler.subscribe(handlersub)
handler.process(event_a)
handler.process(event_b)
handler.process(event_c)
self.assertEqual(len(handlersub.final_history), 1)
self.assertDictEqual(handlersub.final_history[0].events,
{LABEL_LAYER_A: [event_a],
LABEL_LAYER_B: [event_b],
LABEL_LAYER_C: [event_c]})
def test_put_and_get_event(self):
buffer = SharedVersionedMatchBuffer()
run_id = generate_unique_string()
version = RunVersion()
version.add_level(run_id)
buffer.put_event(nfa_name=NFA_NAME_A,
run_id=run_id,
version=version.get_version_as_str(),
state_label=LABEL_LAYER_A,
event=event_a)
self.assertEqual(
event_a,
buffer.get_event(nfa_name=NFA_NAME_A,
state_label=LABEL_LAYER_A,
event_id=event_a.event_id,
default=None))
# incorrect nfa name
self.assertIsNone(
buffer.get_event(nfa_name=NFA_NAME_B,
state_label=LABEL_LAYER_A,
event_id=event_a.event_id,
default=None))
# incorrect state label
self.assertIsNone(
buffer.get_event(nfa_name=NFA_NAME_A,
state_label=LABEL_LAYER_B,
event_id=event_a.event_id,
default=None))
# incorrect event id
self.assertIsNone(
buffer.get_event(nfa_name=NFA_NAME_A,
state_label=LABEL_LAYER_A,
event_id=event_b.event_id,
default=None))
def test_to_dict_1_nfa_1_version_3_events_3_labels(self):
buffer = SharedVersionedMatchBuffer()
run_id_a = generate_unique_string()
version_a = RunVersion()
version_a.add_level(run_id_a)
version_a_str = version_a.get_version_as_str()
# add three events to buffer for version a, run a
buffer.put_event(nfa_name=NFA_NAME_A,
run_id=run_id_a,
version=version_a_str,
state_label=LABEL_LAYER_A,
event=event_a)
buffer.put_event(nfa_name=NFA_NAME_A,
run_id=run_id_a,
version=version_a_str,
state_label=LABEL_LAYER_B,
event=event_b)
buffer.put_event(nfa_name=NFA_NAME_A,
run_id=run_id_a,
version=version_a_str,
state_label=LABEL_LAYER_C,
event=event_c)
# check that it is a dict type
d = buffer.to_dict()
self.assertIsInstance(d, dict)
events = d[SharedVersionedMatchBuffer.EVENTS]
last = d[SharedVersionedMatchBuffer.LAST]
labels = []
# check events
for event in events:
self.assertEqual(NFA_NAME_A,
event[SharedVersionedMatchBuffer.NFA_NAME])
# check that the right event is paired with its label
match_ev = event[SharedVersionedMatchBuffer.MATCH_EVENT]
# check label
label = match_ev[MatchEvent.LABEL]
self.assertTrue((label == LABEL_LAYER_A or label == LABEL_LAYER_B
or label == LABEL_LAYER_C)
and label not in labels)
labels.append(label)
if label == LABEL_LAYER_A:
self.assertDictEqual(event_a.to_dict(),
match_ev[MatchEvent.EVENT])
elif label == LABEL_LAYER_B:
self.assertDictEqual(event_b.to_dict(),
match_ev[MatchEvent.EVENT])
elif label == LABEL_LAYER_C:
self.assertDictEqual(event_c.to_dict(),
match_ev[MatchEvent.EVENT])
match_ev_ids = []
# check last event
for event in last:
self.assertEqual(NFA_NAME_A,
event[SharedVersionedMatchBuffer.NFA_NAME])
self.assertEqual(run_id_a,
event[SharedVersionedMatchBuffer.RUN_ID])
self.assertEqual(version_a_str,
event[SharedVersionedMatchBuffer.VERSION])
match_ev_id = event[SharedVersionedMatchBuffer.EVENT_ID]
self.assertTrue(match_ev_id not in match_ev_ids)
match_ev_ids.append(match_ev_id)