def test_clear_tasks(self):
s = State()
r = ev_snapshot(s)
r.play()
self.assertTrue(s.tasks)
s.clear_tasks(ready=False)
self.assertFalse(s.tasks)
def test_survives_unknown_task_event(self):
s = State()
s.task_event("task-unknown-event-xxx", {
"foo": "bar",
"uuid": "x",
"hostname": "y"
})
def test_clear_tasks(self):
s = State()
r = ev_snapshot(s)
r.play()
assert s.tasks
s.clear_tasks(ready=False)
assert not s.tasks
def test_clear_tasks(self):
s = State()
r = ev_snapshot(s)
r.play()
self.assertTrue(s.tasks)
s.clear_tasks(ready=False)
self.assertFalse(s.tasks)
def test_clear_tasks(self):
s = State()
r = ev_snapshot(s)
r.play()
assert s.tasks
s.clear_tasks(ready=False)
assert not s.tasks
def test_limits_maxtasks(self):
s = State()
s.max_tasks_in_memory = 1
s.task_event('task-unknown-event-xxx', {
'foo': 'bar',
'uuid': 'x',
'hostname': 'y',
'clock': 3
})
s.task_event('task-unknown-event-xxx', {
'foo': 'bar',
'uuid': 'y',
'hostname': 'y',
'clock': 4
})
s.task_event('task-unknown-event-xxx', {
'foo': 'bar',
'uuid': 'z',
'hostname': 'y',
'clock': 5
})
self.assertEqual(len(s._taskheap), 2)
self.assertEqual(s._taskheap[0].clock, 4)
self.assertEqual(s._taskheap[1].clock, 5)
s._taskheap.append(s._taskheap[0])
self.assertTrue(list(s.tasks_by_time()))
def test_survives_unknown_task_event(self):
s = State()
s.task_event('task-unknown-event-xxx', {
'foo': 'bar',
'uuid': 'x',
'hostname': 'y'
})
class Events(threading.Thread):
def __init__(self, capp, **kwargs):
threading.Thread.__init__(self)
self.daemon = True
self.capp = capp
self.state = EventsState(**kwargs)
def run(self):
try_interval = 1
while True:
logger.info("Starting to capture events...")
try:
import celery
try_interval *= 2
with self.capp.connection() as conn:
recv = EventReceiver(conn,
handlers={"*": self.on_event},
app=self.capp)
try_interval = 1
recv.capture(limit=None, timeout=None, wakeup=True)
except Exception as e:
logger.error(
"Failed to capture events: '%s', "
"trying again in %s seconds.", e, try_interval)
logger.debug(e, exc_info=True)
def on_event(self, event):
self.state.event(event)
def test_get_or_create_worker(self):
state = State()
worker, created = state.get_or_create_worker('*****@*****.**')
self.assertEqual(worker.hostname, '*****@*****.**')
self.assertTrue(created)
worker2, created2 = state.get_or_create_worker('*****@*****.**')
self.assertIs(worker2, worker)
self.assertFalse(created2)
def test_get_or_create_task(self):
state = State()
task, created = state.get_or_create_task('id1')
self.assertEqual(task.uuid, 'id1')
self.assertTrue(created)
task2, created2 = state.get_or_create_task('id1')
self.assertIs(task2, task)
self.assertFalse(created2)
def test_get_or_create_task(self):
state = State()
task, created = state.get_or_create_task('id1')
assert task.uuid == 'id1'
assert created
task2, created2 = state.get_or_create_task('id1')
assert task2 is task
assert not created2
def test_get_or_create_worker(self):
state = State()
worker, created = state.get_or_create_worker('*****@*****.**')
self.assertEqual(worker.hostname, '*****@*****.**')
self.assertTrue(created)
worker2, created2 = state.get_or_create_worker('*****@*****.**')
self.assertIs(worker2, worker)
self.assertFalse(created2)
def test_get_or_create_worker(self):
state = State()
worker, created = state.get_or_create_worker('*****@*****.**')
assert worker.hostname == '*****@*****.**'
assert created
worker2, created2 = state.get_or_create_worker('*****@*****.**')
assert worker2 is worker
assert not created2
def test_get_or_create_task(self):
state = State()
task, created = state.get_or_create_task('id1')
assert task.uuid == 'id1'
assert created
task2, created2 = state.get_or_create_task('id1')
assert task2 is task
assert not created2
def test_get_or_create_worker(self):
state = State()
worker, created = state.get_or_create_worker('*****@*****.**')
assert worker.hostname == '*****@*****.**'
assert created
worker2, created2 = state.get_or_create_worker('*****@*****.**')
assert worker2 is worker
assert not created2
def test_get_or_create_task(self):
state = State()
task, created = state.get_or_create_task('id1')
self.assertEqual(task.uuid, 'id1')
self.assertTrue(created)
task2, created2 = state.get_or_create_task('id1')
self.assertIs(task2, task)
self.assertFalse(created2)
def test_thaw__no_replay(self):
s = State()
r = ev_snapshot(s)
s.freeze(buffer=True)
r.play()
s.thaw(replay=False)
self.assertFalse(s.buffer)
self.assertStateEmpty(s)
def test_callback(self):
scratch = {}
def callback(state, event):
scratch['recv'] = True
s = State(callback=callback)
s.event({'type': 'worker-online'})
assert scratch.get('recv')
def test_callback(self):
scratch = {}
def callback(state, event):
scratch["recv"] = True
s = State(callback=callback)
s.event({"type": "worker-online"})
self.assertTrue(scratch.get("recv"))
def test_callback(self):
scratch = {}
def callback(state, event):
scratch["recv"] = True
s = State(callback=callback)
s.event({"type": "worker-online"})
self.assertTrue(scratch.get("recv"))
def test_callback(self):
scratch = {}
def callback(state, event):
scratch['recv'] = True
s = State(callback=callback)
s.event({'type': 'worker-online'})
assert scratch.get('recv')
def test_freeze_while(self):
s = State()
r = ev_snapshot(s)
r.play()
def work():
pass
s.freeze_while(work, clear_after=True)
self.assertFalse(s.event_count)
def test_freeze_while(self):
s = State()
r = ev_snapshot(s)
def work():
r.play()
self.assertStateEmpty(s)
s.freeze_while(work)
self.assertState(s)
def test_freeze_while(self):
s = State()
r = ev_snapshot(s)
r.play()
def work():
pass
s.freeze_while(work, clear_after=True)
self.assertFalse(s.event_count)
def test_survives_unknown_worker_event(self):
s = State()
s.event({
'type': 'worker-unknown-event-xxx',
'foo': 'bar',
})
s.event({
'type': 'worker-unknown-event-xxx',
'hostname': 'xxx',
'foo': 'bar',
})
def test_survives_unknown_task_event(self):
s = State()
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'x',
'hostname': 'y',
'timestamp': time(),
'local_received': time(),
'clock': 0,
})
def test_survives_unknown_task_event(self):
s = State()
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'x',
'hostname': 'y',
'timestamp': time(),
'local_received': time(),
'clock': 0,
})
def test_freeze_thaw__buffering(self):
s = State()
r = ev_snapshot(s)
s.freeze(buffer=True)
self.assertTrue(s._buffering)
r.play()
self.assertStateEmpty(s)
self.assertTrue(s.buffer)
s.thaw()
self.assertState(s)
self.assertFalse(s.buffer)
def test_survives_unknown_task_event(self):
s = State()
s.event(
{
"type": "task-unknown-event-xxx",
"foo": "bar",
"uuid": "x",
"hostname": "y",
"timestamp": time(),
"local_received": time(),
"clock": 0,
}
)
def test_limits_maxtasks(self):
s = State(max_tasks_in_memory=1)
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'x',
'hostname': 'y',
'clock': 3,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'y',
'hostname': 'y',
'clock': 4,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'z',
'hostname': 'y',
'clock': 5,
'timestamp': time(),
'local_received': time(),
})
self.assertEqual(len(s._taskheap), 2)
self.assertEqual(s._taskheap[0].clock, 4)
self.assertEqual(s._taskheap[1].clock, 5)
s._taskheap.append(s._taskheap[0])
self.assertTrue(list(s.tasks_by_time()))
def test_survives_unknown_worker_leaving(self):
s = State(on_node_leave=Mock(name='on_node_leave'))
(worker, created), subject = s.event({
'type': 'worker-offline',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 301030134894833,
})
self.assertEqual(worker, Worker('*****@*****.**'))
self.assertFalse(created)
self.assertEqual(subject, 'offline')
self.assertNotIn('*****@*****.**', s.workers)
s.on_node_leave.assert_called_with(worker)
def test_get_or_create_worker__with_defaults(self):
state = State()
worker, created = state.get_or_create_worker(
'*****@*****.**', pid=30,
)
self.assertEqual(worker.hostname, '*****@*****.**')
self.assertEqual(worker.pid, 30)
self.assertTrue(created)
worker2, created2 = state.get_or_create_worker(
'*****@*****.**', pid=40,
)
self.assertIs(worker2, worker)
self.assertEqual(worker2.pid, 40)
self.assertFalse(created2)
def test_survives_unknown_worker_leaving(self):
s = State(on_node_leave=Mock(name='on_node_leave'))
(worker, created), subject = s.event({
'type': 'worker-offline',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 301030134894833,
})
self.assertEqual(worker, Worker('*****@*****.**'))
self.assertFalse(created)
self.assertEqual(subject, 'offline')
self.assertNotIn('*****@*****.**', s.workers)
s.on_node_leave.assert_called_with(worker)
def test_on_node_join_callback(self):
s = State(on_node_join=Mock(name='on_node_join'))
(worker, created), subject = s.event({
'type': 'worker-online',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 34314,
})
self.assertTrue(worker)
self.assertTrue(created)
self.assertEqual(subject, 'online')
self.assertIn('*****@*****.**', s.workers)
s.on_node_join.assert_called_with(worker)
def test_on_node_join_callback(self):
s = State(on_node_join=Mock(name='on_node_join'))
(worker, created), subject = s.event({
'type': 'worker-online',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 34314,
})
self.assertTrue(worker)
self.assertTrue(created)
self.assertEqual(subject, 'online')
self.assertIn('*****@*****.**', s.workers)
s.on_node_join.assert_called_with(worker)
def test_get_or_create_worker__with_defaults(self):
state = State()
worker, created = state.get_or_create_worker(
'*****@*****.**', pid=30,
)
assert worker.hostname == '*****@*****.**'
assert worker.pid == 30
assert created
worker2, created2 = state.get_or_create_worker(
'*****@*****.**', pid=40,
)
assert worker2 is worker
assert worker2.pid == 40
assert not created2
def test_get_or_create_worker__with_defaults(self):
state = State()
worker, created = state.get_or_create_worker(
'*****@*****.**', pid=30,
)
assert worker.hostname == '*****@*****.**'
assert worker.pid == 30
assert created
worker2, created2 = state.get_or_create_worker(
'*****@*****.**', pid=40,
)
assert worker2 is worker
assert worker2.pid == 40
assert not created2
def test_on_node_join_callback(self):
s = State(on_node_join=Mock(name='on_node_join'))
(worker, created), subject = s.event({
'type': 'worker-online',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 34314,
})
assert worker
assert created
assert subject == 'online'
assert '*****@*****.**' in s.workers
s.on_node_join.assert_called_with(worker)
def test_survives_unknown_worker_leaving(self):
s = State(on_node_leave=Mock(name='on_node_leave'))
(worker, created), subject = s.event({
'type': 'worker-offline',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 301030134894833,
})
assert worker == Worker('*****@*****.**')
assert not created
assert subject == 'offline'
assert '*****@*****.**' not in s.workers
s.on_node_leave.assert_called_with(worker)
def test_on_node_join_callback(self):
s = State(on_node_join=Mock(name='on_node_join'))
(worker, created), subject = s.event({
'type': 'worker-online',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 34314,
})
assert worker
assert created
assert subject == 'online'
assert '*****@*****.**' in s.workers
s.on_node_join.assert_called_with(worker)
def test_survives_unknown_worker_leaving(self):
s = State(on_node_leave=Mock(name='on_node_leave'))
(worker, created), subject = s.event({
'type': 'worker-offline',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 301030134894833,
})
assert worker == Worker('*****@*****.**')
assert not created
assert subject == 'offline'
assert '*****@*****.**' not in s.workers
s.on_node_leave.assert_called_with(worker)
def test_task_children_set_if_received_in_wrong_order(self):
r = ev_task_states(State())
r.events.insert(0, r.events.pop())
r.play()
self.assertIn(r.state.tasks[r.tid2], r.state.tasks[r.tid].children)
self.assertIs(r.state.tasks[r.tid2].root, r.state.tasks[r.tid])
self.assertIs(r.state.tasks[r.tid2].parent, r.state.tasks[r.tid])
def __init__(self,
broker: str,
backend: Optional[str] = None,
max_tasks: Optional[int] = None,
max_workers: Optional[int] = None): # pragma: no cover
"""Construct a Clearly Server instance.
Args:
broker: the broker being used by the celery system
backend: the result backend being used by the celery system
max_tasks: max tasks stored
max_workers: max workers stored
"""
max_tasks, max_workers = max_tasks or 10000, max_workers or 100
logger.info('Creating memory: max_tasks=%d; max_workers=%d', max_tasks,
max_workers)
self._memory = State(max_tasks_in_memory=max_tasks,
max_workers_in_memory=max_workers)
queue_tasks, queue_workers = Queue(), Queue(
) # hands new events to be distributed.
try:
self._listener = EventListener(broker, queue_tasks, queue_workers,
self._memory, backend)
except TimeoutError as e:
logger.critical(e)
sys.exit(1)
self._dispatcher_tasks = StreamingDispatcher(queue_tasks, Role.TASKS)
self._dispatcher_workers = StreamingDispatcher(queue_workers,
Role.WORKERS)
self._rpc = RPCService(self._memory, self._dispatcher_tasks,
self._dispatcher_workers)
def test_get_or_create_worker__with_defaults(self):
state = State()
worker, created = state.get_or_create_worker(
'*****@*****.**',
pid=30,
)
self.assertEqual(worker.hostname, '*****@*****.**')
self.assertEqual(worker.pid, 30)
self.assertTrue(created)
worker2, created2 = state.get_or_create_worker(
'*****@*****.**',
pid=40,
)
self.assertIs(worker2, worker)
self.assertEqual(worker2.pid, 40)
self.assertFalse(created2)
def eventtop():
sys.stderr.write("-> celeryev: starting capture...\n")
state = State()
display = CursesMonitor(state)
display.init_screen()
refresher = DisplayThread(display)
refresher.start()
conn = establish_connection()
recv = EventReceiver(conn, handlers={"*": state.event})
try:
consumer = recv.consumer()
consumer.consume()
while True:
try:
conn.connection.drain_events()
except (socket.timeout, socket.error):
pass
except Exception:
refresher.shutdown = True
refresher.join()
display.resetscreen()
raise
except (KeyboardInterrupt, SystemExit):
conn and conn.close()
refresher.shutdown = True
refresher.join()
display.resetscreen()
def test_task_children_set_if_received_in_wrong_order(self):
r = ev_task_states(State())
r.events.insert(0, r.events.pop())
r.play()
assert r.state.tasks[r.tid2] in r.state.tasks[r.tid].children
assert r.state.tasks[r.tid2].root is r.state.tasks[r.tid]
assert r.state.tasks[r.tid2].parent is r.state.tasks[r.tid]
def __init__(self,
broker,
queue_output,
backend=None,
max_tasks_in_memory=None,
max_workers_in_memory=None):
"""Constructs an event listener instance.
Args:
broker (str): the broker being used by the celery system.
queue_output (Queue): to send to streaming dispatcher.
backend (str): the result backend being used by the celery system.
max_tasks_in_memory (int): max tasks stored
max_workers_in_memory (int): max workers stored
"""
self._app = Celery(broker=broker, backend=backend)
self._queue_output = queue_output
from celery.backends.base import DisabledBackend
self._use_result_backend = not isinstance(self._app.backend,
DisabledBackend)
logger.info('Creating %s: max_tasks=%d; max_workers=%d',
EventListener.__name__, max_tasks_in_memory,
max_workers_in_memory)
logger.info('Celery broker=%s; backend=%s; using_result_backend=%s',
broker, backend, self._use_result_backend)
# events handling: storage and filling missing states.
self.memory = State(
max_tasks_in_memory=max_tasks_in_memory,
max_workers_in_memory=max_workers_in_memory,
) # type: State
# running engine (should be asyncio in the future)
self._listener_thread = None # type:threading.Thread
self._celery_receiver = None # type:EventReceiver
# concurrency control
self._wait_event = threading.Event()
# detect shutdown.
def sigterm_handler(_signo, _stack_frame): # pragma: no cover
self.__stop()
signal.signal(signal.SIGTERM, sigterm_handler)
self.__start()
def test_tasks_by_worker(self):
r = ev_snapshot(State())
r.play()
assert len(list(r.state.tasks_by_worker('utest1'))) == 10
assert len(list(r.state.tasks_by_worker('utest2'))) == 10
assert len(r.state.tasks_by_worker['utest1']) == 10
assert len(r.state.tasks_by_worker['utest2']) == 10
def test_tasks_by_type(self):
r = ev_snapshot(State())
r.play()
assert len(list(r.state.tasks_by_type('task1'))) == 10
assert len(list(r.state.tasks_by_type('task2'))) == 10
assert len(r.state.tasks_by_type['task1']) == 10
assert len(r.state.tasks_by_type['task2']) == 10
def test_worker_heartbeat_expire(self):
r = ev_worker_heartbeats(State())
next(r)
assert not list(r.state.alive_workers())
assert not r.state.workers['utest1'].alive
r.play()
assert list(r.state.alive_workers())
assert r.state.workers['utest1'].alive
def test_worker_online_offline(self):
r = ev_worker_online_offline(State())
next(r)
assert list(r.state.alive_workers())
assert r.state.workers['utest1'].alive
r.play()
assert not list(r.state.alive_workers())
assert not r.state.workers['utest1'].alive
def test_worker_heartbeat_expire(self):
r = ev_worker_heartbeats(State())
next(r)
self.assertFalse(r.state.alive_workers())
self.assertFalse(r.state.workers['utest1'].alive)
r.play()
self.assertTrue(r.state.alive_workers())
self.assertTrue(r.state.workers['utest1'].alive)
def test_task_descending_clock_ordering(self):
state = State()
r = ev_logical_clock_ordering(state)
tA, tB, tC = r.uids
r.play()
now = list(state.tasks_by_time(reverse=False))
assert now[0][0] == tA
assert now[1][0] == tB
assert now[2][0] == tC
for _ in range(1000):
shuffle(r.uids)
tA, tB, tC = r.uids
r.rewind_with_offset(r.current_clock + 1, r.uids)
r.play()
now = list(state.tasks_by_time(reverse=False))
assert now[0][0] == tB
assert now[1][0] == tC
assert now[2][0] == tA
def test_task_logical_clock_ordering(self):
state = State()
r = ev_logical_clock_ordering(state)
tA, tB, tC = r.uids
r.play()
now = list(state.tasks_by_time())
self.assertEqual(now[0][0], tA)
self.assertEqual(now[1][0], tC)
self.assertEqual(now[2][0], tB)
for _ in range(1000):
shuffle(r.uids)
tA, tB, tC = r.uids
r.rewind_with_offset(r.current_clock + 1, r.uids)
r.play()
now = list(state.tasks_by_time())
self.assertEqual(now[0][0], tA)
self.assertEqual(now[1][0], tC)
self.assertEqual(now[2][0], tB)
def test_task_descending_clock_ordering(self):
raise SkipTest('not working')
state = State()
r = ev_logical_clock_ordering(state)
tA, tB, tC = r.uids
r.play()
now = list(state.tasks_by_time(reverse=False))
self.assertEqual(now[0][0], tA)
self.assertEqual(now[1][0], tB)
self.assertEqual(now[2][0], tC)
for _ in range(1000):
shuffle(r.uids)
tA, tB, tC = r.uids
r.rewind_with_offset(r.current_clock + 1, r.uids)
r.play()
now = list(state.tasks_by_time(reverse=False))
self.assertEqual(now[0][0], tB)
self.assertEqual(now[1][0], tC)
self.assertEqual(now[2][0], tA)
def test_limits_maxtasks(self):
s = State(max_tasks_in_memory=1)
s.heap_multiplier = 2
s.event(
{
"type": "task-unknown-event-xxx",
"foo": "bar",
"uuid": "x",
"hostname": "y",
"clock": 3,
"timestamp": time(),
"local_received": time(),
}
)
s.event(
{
"type": "task-unknown-event-xxx",
"foo": "bar",
"uuid": "y",
"hostname": "y",
"clock": 4,
"timestamp": time(),
"local_received": time(),
}
)
s.event(
{
"type": "task-unknown-event-xxx",
"foo": "bar",
"uuid": "z",
"hostname": "y",
"clock": 5,
"timestamp": time(),
"local_received": time(),
}
)
self.assertEqual(len(s._taskheap), 2)
self.assertEqual(s._taskheap[0].clock, 4)
self.assertEqual(s._taskheap[1].clock, 5)
s._taskheap.append(s._taskheap[0])
self.assertTrue(list(s.tasks_by_time()))
def test_limits_maxtasks(self):
s = State()
s.max_tasks_in_memory = 1
s.task_event("task-unknown-event-xxx", {"foo": "bar", "uuid": "x", "hostname": "y", "clock": 3})
s.task_event("task-unknown-event-xxx", {"foo": "bar", "uuid": "y", "hostname": "y", "clock": 4})
s.task_event("task-unknown-event-xxx", {"foo": "bar", "uuid": "z", "hostname": "y", "clock": 5})
self.assertEqual(len(s._taskheap), 2)
self.assertEqual(s._taskheap[0].clock, 4)
self.assertEqual(s._taskheap[1].clock, 5)
s._taskheap.append(s._taskheap[0])
self.assertTrue(list(s.tasks_by_time()))
def test_limits_maxtasks(self):
s = State()
s.max_tasks_in_memory = 1
s.task_event('task-unknown-event-xxx', {'foo': 'bar',
'uuid': 'x',
'hostname': 'y',
'clock': 3})
s.task_event('task-unknown-event-xxx', {'foo': 'bar',
'uuid': 'y',
'hostname': 'y',
'clock': 4})
s.task_event('task-unknown-event-xxx', {'foo': 'bar',
'uuid': 'z',
'hostname': 'y',
'clock': 5})
self.assertEqual(len(s._taskheap), 2)
self.assertEqual(s._taskheap[0].clock, 4)
self.assertEqual(s._taskheap[1].clock, 5)
s._taskheap.append(s._taskheap[0])
self.assertTrue(list(s.tasks_by_time()))
def test_limits_maxtasks(self):
s = State(max_tasks_in_memory=1)
s.heap_multiplier = 2
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'x',
'hostname': 'y',
'clock': 3,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'y',
'hostname': 'y',
'clock': 4,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'z',
'hostname': 'y',
'clock': 5,
'timestamp': time(),
'local_received': time(),
})
assert len(s._taskheap) == 2
assert s._taskheap[0].clock == 4
assert s._taskheap[1].clock == 5
s._taskheap.append(s._taskheap[0])
assert list(s.tasks_by_time())
def test_freeze_while(self):
s = State()
r = ev_snapshot(s)
r.play()
def work():
pass
s.freeze_while(work, clear_after=True)
assert not s.event_count
s2 = State()
r = ev_snapshot(s2)
r.play()
s2.freeze_while(work, clear_after=False)
assert s2.event_count
