def test_repeatable(self):
event = Event()
block = EventSignalTimeout(event)
self.configure_block(block, {
"intervals": [
{
"interval": {
"milliseconds": 200
},
"repeatable": True
}
]
})
block.start()
block.process_signals([Signal({'a': 'A'})])
event.wait(.3)
self.assert_num_signals_notified(1, block)
self.assertDictEqual(block.notified_signals[0].to_dict(),
{'timeout': datetime.timedelta(0, 0, 200000),
'group': 'null',
'a': 'A'})
event.wait(.3)
self.assert_num_signals_notified(2, block)
self.assertDictEqual(block.notified_signals[0].to_dict(),
{'timeout': datetime.timedelta(0, 0, 200000),
'group': 'null',
'a': 'A'})
block.stop()
def test_groups(self, back_patch):
e = Event()
block = EventCounter(e)
self.configure_block(block, {
"reset_info": {
"resetting": True,
"scheme": "INTERVAL",
"interval": {
"seconds": 1
},
},
"group_by": "{{$foo}}"
})
block.start()
block.process_signals([
Signal({'foo': 'bar'}),
Signal({'foo': 'baz'}),
Signal({'qux': 'ly'})
])
self.assertEqual(block._cumulative_count[''], 1)
self.assertEqual(block._cumulative_count['bar'], 1)
self.assertEqual(block._cumulative_count['baz'], 1)
e.wait(2)
for k in block._cumulative_count:
self.assertEqual(block._cumulative_count[k], 0)
block.stop()
def test_groups(self):
event = Event()
block = EventSignalTimeout(event)
self.configure_block(block, {
"intervals": [
{
"interval": {
"milliseconds": 200
},
"repeatable": True
}
],
"group_by": "{{$group}}"
})
block.start()
block.process_signals([Signal({'a': 'A', 'group': 'a'})])
block.process_signals([Signal({'b': 'B', 'group': 'b'})])
# Wait for first notification
event.wait(.3)
self.assert_num_signals_notified(1, block)
self.assertDictEqual(block.notified_signals[0].to_dict(),
{'timeout': datetime.timedelta(0, 0, 200000),
'group': 'a',
'a': 'A'})
# Wait for second notificiation, it should be right after first
event.wait(.3)
self.assert_num_signals_notified(2, block)
self.assertDictEqual(block.notified_signals[0].to_dict(),
{'timeout': datetime.timedelta(0, 0, 200000),
'group': 'b',
'b': 'B'})
block.stop()
def test_all(self, *back_patch):
e1 = Event()
e2 = Event()
e3 = Event()
e4 = Event()
signals = [
EventFlavorSignal(flavor='apple', event=e1),
EventFlavorSignal(flavor='cherry', event=e2),
EventFlavorSignal(flavor='cherry', event=e2),
EventFlavorSignal(flavor='cherry', event=e4)
]
blk = Queue()
config = {
"interval": {
"seconds": 1
},
"capacity": 2,
"group_by": '{{$flavor}}',
"reload": True,
"uniqueness": "{{$flavor}}"
}
self.configure_block(blk, config)
blk.start()
blk.process_signals(signals)
self.assertEqual(len(blk._queues['cherry']), 1)
self.assertEqual(len(blk._queues['apple']), 1)
e1.wait(2)
self.assertEqual(len(blk._queues['cherry']), 1)
self.assertEqual(len(blk._queues['apple']), 1)
blk.process_signals([FlavorSignal('cherry')])
self.assertEqual(len(blk._queues['cherry']), 1)
blk.stop()
def test_reload(self, *back_patch):
e1 = Event()
e2 = Event()
signals = [
EventFlavorSignal(flavor='apple', event=e1),
EventFlavorSignal(flavor='cherry', event=e2)
]
blk = Queue()
config = {
"interval": {
"seconds": 1
},
"capacity": 100,
"group_by": '{{$flavor}}',
"reload": True
}
self.configure_block(blk, config)
blk.start()
blk.process_signals(signals)
self.assertEqual(len(blk._queues['cherry']), 1)
self.assertEqual(len(blk._queues['apple']), 1)
e1.wait(2)
self.assertEqual(len(blk._queues['cherry']), 1)
self.assertEqual(len(blk._queues['apple']), 1)
self.assert_num_signals_notified(2, blk)
blk.stop()
def test_buffer(self):
event = Event()
block = EventBuffer(event)
block._backup = MagicMock()
self.configure_block(block, {"interval": {"milliseconds": 200}})
block.start()
block.process_signals([Signal(), Signal(), Signal(), Signal()])
self.assert_num_signals_notified(0, block)
event.wait(.3)
self.assert_num_signals_notified(4, block)
block.stop()
class IntervalTrigger():
'''Generate signals at a regular interval up to total_signals'''
total_signals = IntProperty(title="Total Number of Signals", default=-1)
interval = TimeDeltaProperty(title='Interval', default={'seconds': 1})
def __init__(self):
super().__init__()
self._stop_event = Event()
self.counter = None
def start(self):
super().start()
self.counter = 0
self._stop_event.clear()
spawn(self.run)
def run(self):
# We'll keep track of when each iteration is expected to finish
interval_secs = self.interval.total_seconds()
expected_finish = time() + interval_secs
while not self._stop_event.is_set():
sigs = self.generate_signals()
# If a generator is returned, build the list
if not isinstance(sigs, list):
sigs = list(sigs)
# Add however many signals were generated (in case multiple
# signals mixin was used) to the counter and notify them
self.counter += len(sigs)
self.notify_signals(sigs)
if self.total_signals > 0 and self.counter >= self.total_signals:
# We have reached our total, stop
break
# sleep the exact correct amount of time
time_remaining = expected_finish - time()
if time_remaining > 0:
# If we have time remaining, sleep until the next iteration
sleep(time_remaining)
# One iteration is complete, increment our next "expected finish"
# time. This expected_finish could fall behind the actual time
# if it takes longer than interval_secs to generate the signals
expected_finish += interval_secs
def stop(self):
""" Stop the simulator thread. """
self._stop_event.set()
super().stop()
def test_with_no_retry(self):
"""All signals are notified and no retry is ever called"""
e = Event()
block = RetryBlock(e, 5)
self.configure_block(block, {})
block.start()
for _ in range(5):
spawn(block.process_signals, [Signal()])
e.wait(2)
block.stop()
self.assert_num_signals_notified(5)
# retry_count was never set because we had no failed calls
self.assertEqual(block._retry_count, None)
def test_successful_retry(self):
"""All signals are notified and a retry is called for each one"""
e = Event()
block = RetryBlock(e, 5)
self.configure_block(block, {})
block.start()
for _ in range(5):
spawn(block.process_signals, [Signal({'error': 1})])
e.wait(2)
block.stop()
self.assert_num_signals_notified(5)
# retry_count is 0 because before_retry was called on 0th fail
self.assertEqual(block._retry_count, 0)
def test_interval_duration(self):
event = Event()
block = EventBuffer(event)
block._backup = MagicMock()
self.configure_block(
block, {
"interval": {
"milliseconds": 1000
},
"interval_duration": {
"milliseconds": 2000
}
})
block.start()
# process 4 signals (first group)
block.process_signals([Signal(), Signal(), Signal(), Signal()])
self.assert_num_signals_notified(0, block)
event.wait(1.3)
# first emit notifies first group
self.assert_num_signals_notified(4, block)
# process 2 more signals (second group)
block.process_signals([Signal(), Signal()])
event.wait(1.3)
# second emit notifies first group and second group
self.assert_num_signals_notified(10, block)
# process 2 more signals (thrid group)
block.process_signals([Signal(), Signal()])
event.wait(1.3)
# third emit notifies second group and third group
self.assert_num_signals_notified(14, block)
block.stop()
def test_timeout(self):
event = Event()
block = EventBuffer(event)
block._backup = MagicMock()
self.configure_block(block, {
"interval": {
"milliseconds": 200
},
"timeout": True
})
block.start()
event.wait(.3)
self.assert_num_signals_notified(1, block)
block.stop()
def test_with_enough_locks(self):
"""All the signals get through since we have enought locks"""
e = Event()
block = LockBlock(e, 5, 5)
self.configure_block(block, {})
block.start()
for _ in range(5):
spawn(block.process_signals, [Signal()])
e.wait(2)
block.stop()
self.assert_num_signals_notified(5)
self.assertEqual(block._number_of_locks, 0)
# All 5 signals are notified and none get blocked
self.assertEqual(block._number_of_lock_queue_full_errors, 0)
# Confirm that all signals were processed
self.assertEqual(block._num_processes_count, 5)
def test_with_one_lock(self):
"""Only the first signal is notified and the others are dropped"""
e = Event()
block = LockBlock(e, 5, 1)
self.configure_block(block, {})
block.start()
for _ in range(5):
spawn(block.process_signals, [Signal()])
e.wait(2)
block.stop()
self.assert_num_signals_notified(1)
self.assertEqual(block._number_of_locks, 0)
# One signal is notifed and 4 are blocked
self.assertEqual(block._number_of_lock_queue_full_errors, 4)
# Confirm that all signals were processed
self.assertEqual(block._num_processes_count, 5)
def test_failed_retry(self):
"""No signals are notified and a retry is called for each one"""
e = Event()
block = RetryBlock(e, 5)
self.configure_block(block, {})
# Let the block retry up to 100 times
block.num_retries = 100
block.start()
for _ in range(5):
spawn(block.process_signals, [Signal({'error': 99})])
e.wait(2)
block.stop()
# No signals get notified since we retry forever
self.assert_num_signals_notified(0)
# Assert that at least 10 retries (2 for each signal)
self.assertTrue(block._retry_count >= 10)
def test_interval_reset(self, back_patch):
e = Event()
block = EventCounter(e)
self.configure_block(block, {
"reset_info": {
"resetting": True,
"scheme": "INTERVAL",
"interval": {
"seconds": 1
}
}
})
block.start()
block.process_signals([Signal(), Signal()])
e.wait(2)
self.assertEqual(block._cumulative_count['null'], 0)
block.stop()
def test_reset(self):
""" Make sure the block can reset the intervals """
event = Event()
block = EventSignalTimeout(event)
self.configure_block(block, {
"intervals": [
{
"interval": {
"seconds": 1
}
}
]
})
block.start()
block.process_signals([Signal({'a': 'A'})])
# Wait a bit before sending another signal
event.wait(0.6)
block.process_signals([Signal({'b': 'B'})])
self.assert_num_signals_notified(0, block)
event.wait(0.6)
self.assert_num_signals_notified(0, block)
event.wait(0.6)
self.assert_num_signals_notified(1, block)
self.assertDictEqual(block.notified_signals[0].to_dict(),
{'timeout': datetime.timedelta(seconds=1),
'group': 'null',
'b': 'B'})
block.stop()
def test_negative_interval(self, *back_patch):
e = Event()
signals = [EventSignal(e)]
blk = Queue()
config = {
"interval": {
"seconds": -1
},
"capacity": 4,
"chunk_size": 1,
}
self.configure_block(blk, config)
blk.start()
blk.process_signals(signals)
e.wait(1)
self.assertEqual(len(blk._queues['null']), 1)
self.assert_num_signals_notified(0, blk)
blk.stop()
def test_cron_missed_reset(self, back_patch):
now = datetime.utcnow()
e = Event()
block = LieCounter(e)
block.reset = MagicMock()
# Reset one minute in the past
reset_at = now - timedelta(minutes=1)
self.configure_block(block, {
"log_level": "DEBUG",
"reset_info": {
"resetting": True,
"scheme": "CRON",
"at": {
"hour": reset_at.hour,
"minute": reset_at.minute
}
},
})
block.start(now - timedelta(minutes=10))
block.process_signals([Signal()])
e.wait(0.5)
block.reset.assert_called_once()
def test_cron_sched(self, back_patch):
now = datetime.utcnow()
e = Event()
block = EventCounter(e)
block._calculate_next = MagicMock(
return_value=now+timedelta(seconds=1))
# Reset one minute in the future
reset_at = now + timedelta(minutes=1)
self.configure_block(block, {
"reset_info": {
"resetting": True,
"scheme": "CRON",
"at": {
"hour": reset_at.hour,
"minute": reset_at.minute
}
},
})
block.start()
block.process_signals([Signal()])
self.assertEqual(block._cumulative_count['null'], 1)
e.wait(1.25)
self.assertEqual(block._cumulative_count['null'], 0)
class SafeTrigger():
""" Guarantees notifying signals every interval, regardless of count """
interval = TimeDeltaProperty(title='Interval', default={'seconds': 1})
max_count = IntProperty(title='Max Count', default=1)
def __init__(self):
super().__init__()
self._job = None
self.stop_event = Event()
self.signal_lock = Lock()
def start(self):
super().start()
self._job = Job(self._emit, self.interval, True)
# Run an emit cycle immediately, but in a new thread since it
# might take some time and we don't want it to hold up start
spawn(self._emit)
def stop(self):
""" Stop the simulator thread and signal generation """
if self._job:
self._job.cancel()
self.stop_event.set()
super().stop()
def _emit(self):
""" Called every *interval* to generate then notify the signals """
self._logger.debug("New generation cycle requested")
count = 0
signals = []
# Stop any currently running simulator threads
self.stop_event.set()
# We only want one simulator thread simulating at a time
with self.signal_lock:
# Ok, we're running, so clear the event and wait
self.stop_event.clear()
self._logger.debug("Starting generation...")
while count < self.max_count and not self.stop_event.is_set():
signals.extend(self.generate_signals(1))
count += 1
self._logger.debug("Notifying {} signals".format(len(signals)))
self.notify_signals(signals)
def __init__(self):
super().__init__()
self._stop_event = Event()
self.counter = None
def test_defaults(self):
e = Event()
blk = self._create_block({}, e)
blk.start()
e.wait(0.1)
blk.stop()
def __init__(self):
super().__init__()
self._job = None
self.stop_event = Event()
self.signal_lock = Lock()
