async def terminal_io_test():
from moler.threaded_moler_connection import ThreadedMolerConnection
received_data = []
moler_conn = ThreadedMolerConnection(
encoder=lambda data: data.encode("utf-8"),
decoder=lambda data: data.decode("utf-8"))
terminal = AsyncioTerminal(moler_connection=moler_conn)
cmds = ['pwd', 'ssh [email protected]', 'password', 'ls\r', 'exit\r']
cmd_idx = [0]
def data_observer(data):
print(data)
received_data.append(data)
print(received_data)
if cmd_idx[0] < len(cmds):
cmd2send = cmds[cmd_idx[0]]
if (cmd2send == 'password') and ('Password' not in data):
return
moler_conn.send(data=cmd2send + '\n')
cmd_idx[0] += 1
moler_conn.subscribe(data_observer)
await terminal.open()
await asyncio.sleep(10)
await terminal.close()
print("end of test")
def test_active_connection_created_from_existing_open_connection_reuses_its_transport(active_sshshell_connection_class):
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection(decoder=lambda data: data.decode("utf-8"),
encoder=lambda data: data.encode("utf-8"))
another_moler_conn = ThreadedMolerConnection(decoder=lambda data: data.decode("utf-8"),
encoder=lambda data: data.encode("utf-8"))
source_connection = active_sshshell_connection_class(moler_connection=moler_conn,
host='localhost', port=22,
username='******', password='******')
with source_connection.open():
source_transport = source_connection._ssh_transport
# no host, port, username, password since we want to create another connection to new shell
# towards same host/port using same credentials
##################################################################################
# CAUTION: they should not share same moler connection (hoever, it is not blocked)
# If they do you should be aware what you are doing
# You are multiplexing io-streams into single moler-connection
##################################################################################
new_connection = active_sshshell_connection_class.from_sshshell(sshshell=source_connection,
moler_connection=another_moler_conn)
assert source_transport is new_connection._ssh_transport
assert new_connection._ssh_transport.is_authenticated() # new one is authenticated
assert new_connection._shell_channel is None # but not open yet (no shell on remote)
with new_connection.open():
assert new_connection._shell_channel is not None
assert source_transport is new_connection._ssh_transport # no change after open()
def ping_observing_task(address):
logger = logging.getLogger('moler.user.app-code')
observer_done = Deferred()
# Lowest layer of Moler's usage (you manually glue all elements):
# 1. create observer
net_down_detector = NetworkDownDetector('10.0.2.15')
# 2. ThreadedMolerConnection is a proxy-glue between observer (speaks str)
# and twisted-connection (speaks bytes)
moler_conn = ThreadedMolerConnection(
decoder=lambda data: data.decode("utf-8"))
# 3a. glue from proxy to observer
moler_conn.subscribe(net_down_detector.data_received)
logger.debug('waiting for data to observe')
def feed_moler(connection_data):
# 3b. glue to proxy from external-IO (twisted tcp client connection)
# (client has to pass it's received data into Moler's connection)
moler_conn.data_received(connection_data)
# 4. Moler's client code must manually check status of observer ...
if net_down_detector.done():
# 5. ... to know when it can ask for result
net_down_time = net_down_detector.result()
timestamp = time.strftime("%H:%M:%S",
time.localtime(net_down_time))
logger.debug('Network is down from {}'.format(timestamp))
observer_done.callback(None) # break tcp client and server
start_tcp_connection(address, feed_moler)
return observer_done
def test_can_receive_binary_data_from_connection(
tcp_connection_class, integration_tcp_server_and_pipe):
from moler.threaded_moler_connection import ThreadedMolerConnection
(tcp_server, tcp_server_pipe) = integration_tcp_server_and_pipe
received_data = bytearray()
receiver_called = threading.Event()
def receiver(data, timestamp):
received_data.extend(data)
receiver_called.set()
def connection_closed_handler():
pass
moler_conn = ThreadedMolerConnection() # no decoder, just pass bytes 1:1
moler_conn.subscribe(receiver,
connection_closed_handler) # build forwarding path
connection = tcp_connection_class(moler_connection=moler_conn,
port=tcp_server.port,
host=tcp_server.host)
with connection.open():
time.sleep(
0.1
) # otherwise we have race between server's pipe and from-client-connection
tcp_server_pipe.send(("send async msg", {'msg': b'data to read'}))
receiver_called.wait(timeout=0.5)
assert b'data to read' == received_data
def test_notifies_only_subscribed_observers_about_data_comming_from_external_io(buffer_transport_class):
from moler.threaded_moler_connection import ThreadedMolerConnection
class BufferObserver(object):
def __init__(self):
self.received_data = []
def on_new_data(self, data, time_recv):
self.received_data.append(data)
buffer_observer1 = BufferObserver()
buffer_observer2 = BufferObserver()
buffer_observer3 = BufferObserver()
moler_conn = ThreadedMolerConnection()
moler_conn.subscribe(observer=buffer_observer1.on_new_data, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=buffer_observer2.on_new_data, connection_closed_handler=do_nothing_func)
used_io = buffer_transport_class(moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"incoming data") # inject to buffer for next line read
used_io.read()
MolerTest.sleep(1, True) # Processing in separate thread so have to wait.
assert b"incoming data" in buffer_observer1.received_data
assert b"incoming data" in buffer_observer2.received_data
assert b"incoming data" not in buffer_observer3.received_data # that one was not subscribed
async def test_can_send_binary_data_over_connection(tcp_connection_class,
integration_tcp_server_and_pipe):
from moler.threaded_moler_connection import ThreadedMolerConnection
(tcp_server, tcp_server_pipe) = integration_tcp_server_and_pipe
moler_conn = ThreadedMolerConnection() # no decoder, just pass bytes 1:1
connection = tcp_connection_class(moler_connection=moler_conn, port=tcp_server.port, host=tcp_server.host)
async with connection:
moler_conn.send(data=b'data to be send') # TODO: await moler_conn.send(data=b'data to be send') ???
time.sleep(0.1) # otherwise we have race between server's pipe and from-client-connection
tcp_server_pipe.send(("get history", {}))
dialog_with_server = tcp_server_pipe.recv()
assert ['Received data:', b'data to be send'] == dialog_with_server[-1]
def test_exception_in_observer_doesnt_break_connection_nor_other_observers(
buffer_transport_class):
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection()
moler_received_data = []
def failing_observer(data):
raise Exception("Fail inside observer")
def one_time_observer(data, time_recv):
moler_received_data.append(data)
moler_conn.unsubscribe(observer=one_time_observer,
connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=failing_observer,
connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=one_time_observer,
connection_closed_handler=do_nothing_func)
used_io = buffer_transport_class(
moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"data 1") # inject to buffer for next line read
used_io.read()
moler_conn.unsubscribe(observer=failing_observer,
connection_closed_handler=do_nothing_func)
MolerTest.sleep(1, True) # Processing in separate thread so have to wait.
assert b"data 1" in moler_received_data
def test_can_notify_its_observer_about_data_comming_from_external_io(buffer_transport_class):
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_received_data = []
def buffer_observer(data, time_recv):
moler_received_data.append(data)
moler_conn = ThreadedMolerConnection()
moler_conn.subscribe(observer=buffer_observer, connection_closed_handler=do_nothing_func)
used_io = buffer_transport_class(moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"incoming data") # inject to buffer for next line read
used_io.read()
MolerTest.sleep(1, True) # Processing in separate thread so have to wait.
assert b"incoming data" in moler_received_data
async def test_wait_for__is_prohibited_inside_async_def(async_runner):
# can't raise in generic runner since why non-async-runner should bother about being used inside 'async def'
# using them in such case is end-user error the same way as using time.sleep(2.41) inside 'async def'
from moler.exceptions import WrongUsage
from moler.threaded_moler_connection import ThreadedMolerConnection
# TODO: can we confidently check "called from async def"
# https://stackoverflow.com/questions/30155138/how-can-i-write-asyncio-coroutines-that-optionally-act-as-regular-functions
# "magically_determine_if_being_yielded_from() is actually event_loop.is_running()"
# but that works for asyncio and not for curio/trio
#
# Any way to treat wait_for() as awaitable?
#
connection_observer = NetworkDownDetector(
connection=ThreadedMolerConnection(), runner=async_runner)
connection_observer.life_status.start_time = time.time(
) # must start observer lifetime before runner.submit()
future = async_runner.submit(connection_observer)
with pytest.raises(WrongUsage) as err:
async_runner.wait_for(connection_observer, future)
connection_observer.result() # should raise WrongUsage
assert "Can't call wait_for() from 'async def' - it is blocking call" in str(
err.value)
# check "fix-hint" inside exception
assert re.findall(
r'consider using:\s+await observer\s+instead of:\s+observer.await_done()',
str(err.value))
def test_runner_secures_observer_against_additional_data_after_runner_shutdown(
observer_runner):
"""In-shutdown runner should not pass data to observer even before unsubscribe from moler-connection"""
# Even without running background feeder
# we can use correctly constructed secure.data_received(data, datetime.datetime.now())
# to block passing data from connection to observer while runner is in-shutdown state
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection()
# check if shutdown stops all observers running inside given runner
net_down_detector1 = NetworkDownDetector(connection=moler_conn,
runner=observer_runner)
net_down_detector2 = NetworkDownDetector(connection=moler_conn,
runner=observer_runner)
net_down_detector1.life_status.start_time = time.time(
) # must start observer lifetime before runner.submit()
net_down_detector2.life_status.start_time = time.time(
) # must start observer lifetime before runner.submit()
connection = moler_conn
observer_runner.submit(net_down_detector1)
observer_runner.submit(net_down_detector2)
connection.data_received("61 bytes", datetime.datetime.now())
observer_runner.shutdown()
connection.data_received("62 bytes", datetime.datetime.now())
assert net_down_detector1.all_data_received == ["61 bytes"]
assert net_down_detector2.all_data_received == ["61 bytes"]
def test_can_create_active_sshshell_connection_using_same_api(active_sshshell_connection_class):
# we want to have all connections of class 'active sshshell' to share same API
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection(decoder=lambda data: data.decode("utf-8"),
encoder=lambda data: data.encode("utf-8"))
connection = active_sshshell_connection_class(moler_connection=moler_conn,
host='localhost', port=22,
username='******', password='******')
assert connection._ssh_transport is None
assert connection._shell_channel is None
assert hasattr(connection, "data_received")
# API allows to use "login" as alias for "username" parameter (to keep parity with ssh command / OpenSSH)
connection1 = active_sshshell_connection_class(moler_connection=moler_conn,
host='localhost', port=22,
login='******', password='******')
assert connection1._ssh_transport is None
assert connection1._shell_channel is None
assert hasattr(connection1, "data_received")
assert connection1.sshshell.username == 'molerssh'
# but you shouldn't use both names, we can't guess which one you wanted
with pytest.raises(KeyError) as err:
active_sshshell_connection_class(moler_connection=moler_conn, host='localhost', port=22,
username='******', login='******', password='******')
assert "Use either 'username' or 'login', not both" in str(err.value)
def test_opening_connection_created_from_existing_one_is_quicker(sshshell_connection):
from moler.threaded_moler_connection import ThreadedMolerConnection
connection = sshshell_connection
if hasattr(connection, "moler_connection"): # active connection
another_moler_conn = ThreadedMolerConnection(decoder=lambda data: data.decode("utf-8"),
encoder=lambda data: data.encode("utf-8"))
new_connection = sshshell_connection.__class__.from_sshshell(moler_connection=another_moler_conn,
sshshell=connection)
else:
new_connection = sshshell_connection.__class__.from_sshshell(sshshell=connection)
full_open_durations = []
reused_conn_open_durations = []
for cnt in range(10):
start1 = time.time()
with connection.open():
end1 = time.time()
start2 = time.time()
with new_connection.open():
end2 = time.time()
full_open_durations.append(end1 - start1)
reused_conn_open_durations.append(end2 - start2)
avr_full_open_duration = sum(full_open_durations) / len(full_open_durations)
avr_reused_conn_open_duration = sum(reused_conn_open_durations) / len(reused_conn_open_durations)
assert (avr_reused_conn_open_duration * 3) < avr_full_open_duration
def test_events_true_any_one():
connection = ThreadedMolerConnection()
events = list()
patterns = ("aaa", "bbb")
for pattern in patterns:
event = Wait4prompt(connection=connection,
till_occurs_times=1,
prompt=pattern)
event.start()
events.append(event)
connection.data_received(patterns[0], datetime.datetime.now())
assert EventAwaiter.wait_for_any(timeout=0.1, events=events) is True
done, not_done = EventAwaiter.separate_done_events(events)
assert 1 == len(done)
assert 1 == len(not_done)
EventAwaiter.cancel_all_events(events)
def tcp_thd_conn(port, host='localhost', name=None):
moler_conn = ThreadedMolerConnection(decoder=lambda data: data.decode("utf-8"))
conn_logger_name = 'threaded.tcp-connection({}:{})'.format(host, port)
conn_logger = logging.getLogger(conn_logger_name)
io_conn = tcp.ThreadedTcp(moler_connection=moler_conn,
port=port, host=host, logger=conn_logger)
return io_conn
def memory_connection(memory_connection_class):
connection_class = memory_connection_class
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection(decoder=lambda data: data.decode("utf-8"),
encoder=lambda data: data.encode("utf-8"))
connection = connection_class(moler_connection=moler_conn)
return connection
def test_repr_conversion_of_command_object():
"""
repr() conversion shows same as str() plus embedded connection used by command
"""
moler_conn = ThreadedMolerConnection(decoder=lambda data: data.decode("utf-8"))
class LsCmd(Command):
def __init__(self, options='-l', connection=None):
super(LsCmd, self).__init__(connection=connection)
self.command_string = 'ls {}'.format(options)
def data_received(self, data, recv_time):
pass # not important now
ls = LsCmd(connection=moler_conn)
# (1) command with ThreadedMolerConnection to glued to ext-io
assert 'LsCmd("ls -l", id:{}, using ThreadedMolerConnection(id:{})-->[?])'.format(instance_id(ls), instance_id(moler_conn)) == repr(ls)
# TODO: add test for <ThreadedMolerConnection( id:{}>
# (2) command with ThreadedMolerConnection glued to ext-io
ext_io_connection = FifoBuffer(moler_connection=moler_conn)
how2send_repr = repr(ext_io_connection.write)
assert 'LsCmd("ls -l", id:{}, using ThreadedMolerConnection(id:{})-->[{}])'.format(instance_id(ls), instance_id(moler_conn), how2send_repr) == repr(ls)
# TODO: move ThreadedMolerConnection(id:{})-->[{}])'.format(instance_id(moler_conn), how2send_repr) into ThreadedMolerConnection __repr__ test
# TODO: and here just:
# assert 'LsCmd("ls -l", id:{}, using {})'.format(instance_id(ls), repr(moler_conn)) == repr(ls)
# (3) command without connection
ls.connection = None
assert 'LsCmd("ls -l", id:{}, using <NO CONNECTION>)'.format(instance_id(ls)) == repr(ls)
def connection_observer(observer_runner):
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection()
observer = NetworkDownDetector(connection=moler_conn, runner=observer_runner)
yield observer
# remove exceptions collected inside ConnectionObserver
ConnectionObserver.get_unraised_exceptions(remove=True)
def test_command_string_is_required_to_call_command(command_major_base_class):
import threading
from moler.exceptions import NoCommandStringProvided
moler_conn = ThreadedMolerConnection()
command_class = do_nothing_command_class(base_class=command_major_base_class)
command = command_class(connection=moler_conn)
assert not command.command_string # ensure it is empty before starting command
def command_in_thread():
with pytest.raises(NoCommandStringProvided) as error:
command()
assert error.value.command == command
assert 'for {}'.format(str(command)) in str(error.value)
assert 'You should fill .command_string member before starting command' in str(error.value)
cmd_thrd = threading.Thread(target=command_in_thread)
cmd_thrd.start()
cmd_thrd.join()
command = command_class(connection=moler_conn)
with pytest.raises(NoCommandStringProvided) as error:
command() # call the command-future (foreground run)
assert error.value.command == command
assert 'for {}'.format(str(command)) in str(error.value)
assert 'You should fill .command_string member before starting command' in str(error.value)
def ext_io_connection(request):
from moler.io.raw.memory import ThreadedFifoBuffer
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection(
decoder=lambda data: data.decode("utf-8"))
connection = ThreadedFifoBuffer(moler_connection=moler_conn)
return connection
def test_runner_shutdown_cancels_remaining_feeders_inside_threads(observer_runner):
from moler.threaded_moler_connection import ThreadedMolerConnection
observers_pool = []
for idx in range(3):
connection_observer = NetworkDownDetector(connection=ThreadedMolerConnection(), runner=observer_runner)
observers_pool.append(connection_observer)
def submit_feeder(connection_observer):
connection_observer.life_status.start_time = time.time() # must start observer lifetime before runner.submit()
future = observer_runner.submit(connection_observer)
while not future.done():
time.sleep(0.1)
th_pool = [threading.Thread(target=submit_feeder, args=(connection_observer,)) for connection_observer in observers_pool]
for th in th_pool:
th.start()
# loop.run_until_complete(remaining_tasks) # let it enter feeder
time.sleep(0.5)
observer_runner.shutdown()
for th in th_pool:
th.join()
assert observers_pool[0].cancelled()
assert observers_pool[1].cancelled()
assert observers_pool[2].cancelled()
def test_runner_secures_observer_against_additional_data_after_observer_is_done(observer_runner):
"""Done observer should not get data even before unsubscribe from moler-connection"""
# correctly written observer looks like:
#
# def data_received(self, data, recv_time):
# if not self.done():
# parse(data)
#
# This test checks if runners secure wrong-written-observers with missing 'if not self.done():'
from moler.threaded_moler_connection import ThreadedMolerConnection
with disabled_logging():
for n in range(20): # need to test multiple times to ensure there are no thread races
moler_conn = ThreadedMolerConnection()
net_down_detector = NetworkDownDetector(connection=moler_conn, runner=observer_runner)
net_down_detector.life_status.start_time = time.time() # must start observer lifetime before runner.submit()
connection = net_down_detector.connection
net_down_detector.life_status.start_time = time.time()
observer_runner.submit(net_down_detector)
connection.data_received("61 bytes", datetime.datetime.now())
connection.data_received("ping: Network is unreachable", datetime.datetime.now())
connection.data_received("62 bytes", datetime.datetime.now())
assert net_down_detector.all_data_received == ["61 bytes", "ping: Network is unreachable"]
async def test_observer_gets_all_data_of_connection_after_it_is_submitted_to_background(observer_runner):
# another words: after returning from runner.submit() no data can be lost, no races
# Raw 'def' usage note:
# This functionality works as well when runner is used inside raw def function
# since it only uses runner.submit() + awaiting time
# another words - runner is running over some time period
# The only difference is that raw def function may use only standalone_runner (which is subset of observer_runner)
# and inside test you exchange 'await asyncio.sleep()' with 'time.sleep()'
from moler.threaded_moler_connection import ThreadedMolerConnection
with disabled_logging():
durations = []
for n in range(20): # need to test multiple times to ensure there are no thread races
moler_conn = ThreadedMolerConnection()
net_down_detector = NetworkDownDetector(connection=moler_conn, runner=observer_runner)
connection = net_down_detector.connection
start_time = net_down_detector.life_status.start_time = time.time()
observer_runner.submit(net_down_detector)
durations.append(time.time() - start_time)
connection.data_received("61 bytes", datetime.datetime.now())
connection.data_received("62 bytes", datetime.datetime.now())
connection.data_received("ping: Network is unreachable", datetime.datetime.now())
assert net_down_detector.all_data_received == ["61 bytes", "62 bytes", "ping: Network is unreachable"]
print("\n{}.submit() duration == {}".format(observer_runner.__class__.__name__,
float(sum(durations))/len(durations)))
def terminal_connection():
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection()
terminal = ThreadedTerminal(moler_connection=moler_conn)
with terminal.open() as connection:
yield connection.moler_connection
def active_sshshell_connection(active_sshshell_connection_class):
from moler.threaded_moler_connection import ThreadedMolerConnection
connection_class = active_sshshell_connection_class
moler_conn = ThreadedMolerConnection(decoder=lambda data: data.decode("utf-8"),
encoder=lambda data: data.encode("utf-8"))
connection = connection_class(moler_connection=moler_conn,
host='localhost', port=22, username='******', password='******')
return connection
def conn_observer(request):
moler_conn = ThreadedMolerConnection(how2send=mock.MagicMock())
if request.param == 'generic_observer':
observer = NetworkDownDetector(connection=moler_conn)
elif request.param == 'event':
observer = MyEvent(connection=moler_conn)
elif request.param == 'command':
observer = MyCommand(connection=moler_conn)
return observer
def test_notified_observer_may_stop_subscription_of_data_comming_from_external_io(buffer_transport_class):
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection()
moler_received_data = []
def one_time_observer(data, time_recv):
moler_received_data.append(data)
moler_conn.unsubscribe(observer=one_time_observer, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=one_time_observer, connection_closed_handler=do_nothing_func)
used_io = buffer_transport_class(moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"data 1") # inject to buffer for next line read
used_io.read()
used_io.write(input_bytes=b"data 2") # inject to buffer for next line read
used_io.read()
MolerTest.sleep(1, True) # Processing in separate thread so have to wait.
assert b"data 1" in moler_received_data
assert b"data 2" not in moler_received_data # because of unsubscription during notification
def test_connection_has_running_loop_after_open(
tcp_connection_class, integration_tcp_server_and_pipe):
from moler.threaded_moler_connection import ThreadedMolerConnection
(tcp_server, tcp_server_pipe) = integration_tcp_server_and_pipe
moler_conn = ThreadedMolerConnection()
connection = tcp_connection_class(moler_connection=moler_conn,
port=tcp_server.port,
host=tcp_server.host)
connection.open()
assert connection._async_tcp._stream_reader._loop.is_running()
def test_runner_shutdown_cancels_remaining_inactive_feeders_inside_main_thread(observer_runner):
from moler.threaded_moler_connection import ThreadedMolerConnection
connection_observer = NetworkDownDetector(connection=ThreadedMolerConnection(), runner=observer_runner)
connection_observer.life_status.start_time = time.time() # must start observer lifetime before runner.submit()
future = observer_runner.submit(connection_observer)
time.sleep(0.2) # won't enter event loop of future - feeder won't start processing
observer_runner.shutdown()
assert connection_observer.cancelled()
def test_event_string_is_required_to_start_command(lineevent_class):
from moler.exceptions import NoDetectPatternProvided
moler_conn = ThreadedMolerConnection()
event_class = do_nothing_connection_observer_class(
base_class=lineevent_class)
event = event_class(connection=moler_conn, detect_patterns=[])
assert not event.detect_patterns # ensure it is empty before starting command
with pytest.raises(NoDetectPatternProvided) as error:
event.start() # start the command-future (background run)
def test_runner_shutdown_cancels_remaining_active_feeders_inside_main_thread(async_runner):
from moler.threaded_moler_connection import ThreadedMolerConnection
connection_observer = NetworkDownDetector(connection=ThreadedMolerConnection(), runner=async_runner)
connection_observer.life_status.start_time = time.time() # must start observer lifetime before runner.submit()
future = async_runner.submit(connection_observer)
future._loop.run_until_complete(asyncio.sleep(1.0)) # feeder will start processing inside loop
# time.sleep(0.5)
async_runner.shutdown()
assert connection_observer.cancelled()
