def test_can_receive_binary_data_from_connection(
tcp_connection_class, integration_tcp_server_and_pipe):
from moler.threaded_moler_connection import ThreadedMolerConnection
(tcp_server, tcp_server_pipe) = integration_tcp_server_and_pipe
received_data = bytearray()
receiver_called = threading.Event()
def receiver(data, timestamp):
received_data.extend(data)
receiver_called.set()
def connection_closed_handler():
pass
moler_conn = ThreadedMolerConnection() # no decoder, just pass bytes 1:1
moler_conn.subscribe(receiver,
connection_closed_handler) # build forwarding path
connection = tcp_connection_class(moler_connection=moler_conn,
port=tcp_server.port,
host=tcp_server.host)
with connection.open():
time.sleep(
0.1
) # otherwise we have race between server's pipe and from-client-connection
tcp_server_pipe.send(("send async msg", {'msg': b'data to read'}))
receiver_called.wait(timeout=0.5)
assert b'data to read' == received_data
async def ping_observing_task(address):
logger = logging.getLogger('moler.user.app-code')
# Lowest layer of Moler's usage (you manually glue all elements):
# 1. create observer
net_down_detector = NetworkDownDetector('10.0.2.15')
# 2. ThreadedMolerConnection is a proxy-glue between observer (speaks str)
# and asyncio-connection (speaks bytes)
moler_conn = ThreadedMolerConnection(
decoder=lambda data: data.decode("utf-8"))
# 3a. glue from proxy to observer
moler_conn.subscribe(net_down_detector.data_received)
logger.debug('waiting for data to observe')
async for connection_data in tcp_connection(address):
# 3b. glue to proxy from external-IO (asyncio tcp client connection)
# (client code has to pass it's received data into Moler's connection)
moler_conn.data_received(connection_data)
# 4. Moler's client code must manually check status of observer ...
if net_down_detector.done():
# 5. ... to know when it can ask for result
net_down_time = net_down_detector.result()
timestamp = time.strftime("%H:%M:%S",
time.localtime(net_down_time))
logger.debug('Network is down from {}'.format(timestamp))
break
def test_repeated_unsubscription_does_nothing_but_logs_warning(buffer_transport_class):
"""
Because of possible different concurrency models (and their races)
we don't want to raise exception when there is already
"no such subscription" - just put warning to logs
"""
import time
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection()
moler_received_data = []
def one_time_observer(data, time_recv):
moler_received_data.append(data)
moler_conn.unsubscribe(observer=one_time_observer, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=one_time_observer, connection_closed_handler=do_nothing_func)
# subscribe fires new thread via ObserverThreadWrapper
# that thread pushes data: moler-connection.notify_observers() --> queue --> one_time_observer()
used_io = buffer_transport_class(moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"data 1") # inject to buffer for next line read
used_io.read()
time.sleep(0.2) # allow threads switch to let ObserverThreadWrapper push data into one_time_observer()
moler_conn.unsubscribe(observer=one_time_observer, connection_closed_handler=do_nothing_func) # TODO: check
# warning in logs (when we set logging system)
used_io.write(input_bytes=b"data 2") # inject to buffer for next line read
used_io.read()
MolerTest.sleep(1, True) # Processing in separate thread so have to wait.
assert b"data 1" in moler_received_data
assert b"data 2" not in moler_received_data # because of unsubscription during notification
def test_notifies_only_subscribed_observers_about_data_comming_from_external_io(buffer_transport_class):
from moler.threaded_moler_connection import ThreadedMolerConnection
class BufferObserver(object):
def __init__(self):
self.received_data = []
def on_new_data(self, data, time_recv):
self.received_data.append(data)
buffer_observer1 = BufferObserver()
buffer_observer2 = BufferObserver()
buffer_observer3 = BufferObserver()
moler_conn = ThreadedMolerConnection()
moler_conn.subscribe(observer=buffer_observer1.on_new_data, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=buffer_observer2.on_new_data, connection_closed_handler=do_nothing_func)
used_io = buffer_transport_class(moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"incoming data") # inject to buffer for next line read
used_io.read()
MolerTest.sleep(1, True) # Processing in separate thread so have to wait.
assert b"incoming data" in buffer_observer1.received_data
assert b"incoming data" in buffer_observer2.received_data
assert b"incoming data" not in buffer_observer3.received_data # that one was not subscribed
async def terminal_io_test():
from moler.threaded_moler_connection import ThreadedMolerConnection
received_data = []
moler_conn = ThreadedMolerConnection(
encoder=lambda data: data.encode("utf-8"),
decoder=lambda data: data.decode("utf-8"))
terminal = AsyncioTerminal(moler_connection=moler_conn)
cmds = ['pwd', 'ssh [email protected]', 'password', 'ls\r', 'exit\r']
cmd_idx = [0]
def data_observer(data):
print(data)
received_data.append(data)
print(received_data)
if cmd_idx[0] < len(cmds):
cmd2send = cmds[cmd_idx[0]]
if (cmd2send == 'password') and ('Password' not in data):
return
moler_conn.send(data=cmd2send + '\n')
cmd_idx[0] += 1
moler_conn.subscribe(data_observer)
await terminal.open()
await asyncio.sleep(10)
await terminal.close()
print("end of test")
def test_exception_in_observer_doesnt_break_connection_nor_other_observers(
buffer_transport_class):
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection()
moler_received_data = []
def failing_observer(data):
raise Exception("Fail inside observer")
def one_time_observer(data, time_recv):
moler_received_data.append(data)
moler_conn.unsubscribe(observer=one_time_observer,
connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=failing_observer,
connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=one_time_observer,
connection_closed_handler=do_nothing_func)
used_io = buffer_transport_class(
moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"data 1") # inject to buffer for next line read
used_io.read()
moler_conn.unsubscribe(observer=failing_observer,
connection_closed_handler=do_nothing_func)
MolerTest.sleep(1, True) # Processing in separate thread so have to wait.
assert b"data 1" in moler_received_data
def test_can_notify_its_observer_about_data_comming_from_external_io(buffer_transport_class):
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_received_data = []
def buffer_observer(data, time_recv):
moler_received_data.append(data)
moler_conn = ThreadedMolerConnection()
moler_conn.subscribe(observer=buffer_observer, connection_closed_handler=do_nothing_func)
used_io = buffer_transport_class(moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"incoming data") # inject to buffer for next line read
used_io.read()
MolerTest.sleep(1, True) # Processing in separate thread so have to wait.
assert b"incoming data" in moler_received_data
def test_notified_observer_may_stop_subscription_of_data_comming_from_external_io(buffer_transport_class):
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection()
moler_received_data = []
def one_time_observer(data, time_recv):
moler_received_data.append(data)
moler_conn.unsubscribe(observer=one_time_observer, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=one_time_observer, connection_closed_handler=do_nothing_func)
used_io = buffer_transport_class(moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"data 1") # inject to buffer for next line read
used_io.read()
used_io.write(input_bytes=b"data 2") # inject to buffer for next line read
used_io.read()
MolerTest.sleep(1, True) # Processing in separate thread so have to wait.
assert b"data 1" in moler_received_data
assert b"data 2" not in moler_received_data # because of unsubscription during notification
def test_garbage_collected_subscriber_is_not_notified():
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection()
received_data = []
class Subscriber(object):
def __call__(self, data, time_recv):
received_data.append(data)
subscr1 = Subscriber()
subscr2 = Subscriber()
moler_conn.subscribe(observer=subscr1, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=subscr2, connection_closed_handler=do_nothing_func)
del subscr1
gc.collect()
moler_conn.data_received("data", datetime.datetime.now())
MolerTest.sleep(1, True) # Processing in separate thread so have to wait.
assert len(received_data) == 1
def test_subscription_doesnt_block_subscriber_to_be_garbage_collected():
from moler.threaded_moler_connection import ThreadedMolerConnection
moler_conn = ThreadedMolerConnection()
garbage_collected_subscribers = []
class Subscriber(object):
def __del__(self):
garbage_collected_subscribers.append('Subscriber')
class CloseSubscriber(object):
pass
subscr = Subscriber()
close_subscr = CloseSubscriber()
moler_conn.subscribe(subscr, close_subscr)
del subscr
gc.collect()
assert 'Subscriber' in garbage_collected_subscribers
def ping_observing_task(address, ping_ip):
logger = logging.getLogger('moler.user.app-code')
net_addr = 'tcp://{}:{}'.format(*address)
# Lowest layer of Moler's usage (you manually glue all elements):
# 1. create observers
net_down_detector = NetworkDownDetector(ping_ip)
net_drop_found = False
net_up_detector = NetworkUpDetector(ping_ip)
moler_conn = ThreadedMolerConnection(
decoder=lambda data: data.decode("utf-8"))
# 2. virtually "start" observer by making it data-listener
moler_conn.subscribe(net_down_detector.data_received)
info = '{} on {} using {}'.format(ping_ip, net_addr, net_down_detector)
logger.debug('observe ' + info)
for _ in tcp_connection(address, moler_conn):
# anytime new data comes it may change status of observer
if not net_drop_found and net_down_detector.done():
net_drop_found = True
net_down_time = net_down_detector.result()
timestamp = time.strftime("%H:%M:%S",
time.localtime(net_down_time))
logger.debug('Network {} is down from {}'.format(
ping_ip, timestamp))
# 3. virtually "stop" that observer
moler_conn.unsubscribe(net_down_detector.data_received)
# 4. and start subsequent one (to know when net is back "up")
info = '{} on {} using {}'.format(ping_ip, net_addr,
net_up_detector)
logger.debug('observe ' + info)
moler_conn.subscribe(net_up_detector.data_received)
if net_up_detector.done():
net_up_time = net_up_detector.result()
timestamp = time.strftime("%H:%M:%S", time.localtime(net_up_time))
logger.debug('Network {} is back "up" from {}'.format(
ping_ip, timestamp))
# 5. virtually "stop" that observer
moler_conn.unsubscribe(net_up_detector.data_received)
break
def test_single_unsubscription_doesnt_impact_other_subscribers():
from moler.threaded_moler_connection import ThreadedMolerConnection
class TheObserver(object):
def __init__(self):
self.received_data = []
def on_new_data(self, data, time_recv):
self.received_data.append(data)
observer1 = TheObserver()
observer2 = TheObserver()
function_received_data = []
def raw_fun1(data, time_recv):
function_received_data.append(data)
def raw_fun2(data, time_recv):
function_received_data.append(data)
class TheCallableClass(object):
def __init__(self):
self.received_data = []
def __call__(self, data, time_recv):
self.received_data.append(data)
callable1 = TheCallableClass()
callable2 = TheCallableClass()
moler_conn = ThreadedMolerConnection()
moler_conn.subscribe(observer=observer1.on_new_data, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=observer2.on_new_data, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=observer2.on_new_data, connection_closed_handler=do_nothing_func)
moler_conn.unsubscribe(observer=observer1.on_new_data, connection_closed_handler=do_nothing_func)
moler_conn.unsubscribe(observer=observer1.on_new_data, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=raw_fun1, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=raw_fun2, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=raw_fun2, connection_closed_handler=do_nothing_func)
moler_conn.unsubscribe(observer=raw_fun1, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=callable1, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=callable2, connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=callable2, connection_closed_handler=do_nothing_func)
moler_conn.unsubscribe(observer=callable1, connection_closed_handler=do_nothing_func)
moler_conn.data_received("incoming data", datetime.datetime.now())
MolerTest.sleep(1, True) # Processing in separate thread so have to wait.
assert observer1.received_data == []
assert observer2.received_data == ["incoming data"]
assert function_received_data == ["incoming data"]
assert callable1.received_data == []
assert callable2.received_data == ["incoming data"]
def test_can_work_with_multiple_connections(
tcp_connection_class, integration_tcp_server_and_pipe,
integration_second_tcp_server_and_pipe):
"""Check open/close/send/receive on multiple connections"""
from moler.threaded_moler_connection import ThreadedMolerConnection
(tcp_server0, tcp_server0_pipe) = integration_tcp_server_and_pipe
(tcp_server1, tcp_server1_pipe) = integration_second_tcp_server_and_pipe
received_data = [bytearray(), bytearray()]
receiver_called = [threading.Event(), threading.Event()]
def receiver0(data, time_recv):
received_data[0].extend(data)
receiver_called[0].set()
def receiver1(data, time_recv):
received_data[1].extend(data)
receiver_called[1].set()
def do_nothig_func():
pass
moler_conn0 = ThreadedMolerConnection()
moler_conn0.subscribe(receiver0, do_nothig_func)
moler_conn1 = ThreadedMolerConnection()
moler_conn1.subscribe(receiver1, do_nothig_func)
connection0 = tcp_connection_class(moler_connection=moler_conn0,
port=tcp_server0.port,
host=tcp_server0.host)
connection1 = tcp_connection_class(moler_connection=moler_conn1,
port=tcp_server1.port,
host=tcp_server1.host)
with connection0.open():
with connection1.open():
time.sleep(0.1) # to let servers notify connecting clients
tcp_server0_pipe.send(("send async msg", {
'msg': b'data from server 0'
}))
tcp_server1_pipe.send(("send async msg", {
'msg': b'data from server 1'
}))
assert receiver_called[0].wait(timeout=0.5)
assert receiver_called[1].wait(timeout=0.5)
moler_conn0.send(data=b'data to server 0')
moler_conn1.send(data=b'data to server 1')
time.sleep(0.1) # to let servers get what was sent
# what we got from servers
assert b'data from server 0' == received_data[0]
assert b'data from server 1' == received_data[1]
# what servers know about clients
tcp_server0_pipe.send(("get history", {}))
tcp_server1_pipe.send(("get history", {}))
dialog_with_server0 = tcp_server0_pipe.recv()
dialog_with_server1 = tcp_server1_pipe.recv()
assert 'Client connected' == dialog_with_server0[0]
assert 'Client connected' == dialog_with_server0[0]
assert ['Received data:', b'data to server 0'] == dialog_with_server0[-2]
assert ['Received data:', b'data to server 1'] == dialog_with_server1[-2]
assert 'Client disconnected' == dialog_with_server0[-1]
assert 'Client disconnected' == dialog_with_server1[-1]
