async def test_can_send_binary_data_over_connection(
tcp_connection_class, integration_tcp_server_and_pipe):
from moler.connection import ObservableConnection
(tcp_server, tcp_server_pipe) = integration_tcp_server_and_pipe
moler_conn = ObservableConnection() # no decoder, just pass bytes 1:1
connection = tcp_connection_class(moler_connection=moler_conn,
port=tcp_server.port,
host=tcp_server.host)
async with connection:
moler_conn.send(
data=b'data to be send'
) # TODO: await moler_conn.send(data=b'data to be send') ???
time.sleep(
0.1
) # otherwise we have race between server's pipe and from-client-connection
tcp_server_pipe.send(("get history", {}))
dialog_with_server = tcp_server_pipe.recv()
assert ['Received data:', b'data to be send'] == dialog_with_server[-1]
def tcp_thd_conn(port, host='localhost', name=None):
moler_conn = ObservableConnection(
decoder=lambda data: data.decode("utf-8"))
conn_logger_name = 'threaded.tcp-connection({}:{})'.format(host, port)
conn_logger = logging.getLogger(conn_logger_name)
io_conn = tcp.ThreadedTcp(moler_connection=moler_conn,
port=port,
host=host,
logger=conn_logger)
return io_conn
def test_event_string_is_required_to_start_command(lineevent_class):
from moler.exceptions import NoDetectPatternProvided
moler_conn = ObservableConnection()
event_class = do_nothing_command_class(base_class=lineevent_class)
event = event_class(connection=moler_conn, detect_patterns=[])
assert not event.detect_patterns # ensure it is empty before starting command
with pytest.raises(NoDetectPatternProvided) as error:
event.start() # start the command-future (background run)
def test_connections_use_same_loop(tcp_connection_class,
integration_tcp_server_and_pipe,
integration_second_tcp_server_and_pipe):
# same loop means also same thread since asyncio has one loop in thread
from moler.connection import ObservableConnection
(tcp_server0, tcp_server0_pipe) = integration_tcp_server_and_pipe
(tcp_server1, tcp_server1_pipe) = integration_second_tcp_server_and_pipe
connection0 = tcp_connection_class(moler_connection=ObservableConnection(),
port=tcp_server0.port, host=tcp_server0.host)
connection1 = tcp_connection_class(moler_connection=ObservableConnection(),
port=tcp_server1.port, host=tcp_server1.host)
with connection0.open():
with connection1.open():
# loop and thread appear after open()
async_loop_of_connection0 = connection0._async_tcp._stream_reader._loop
async_loop_of_connection1 = connection0._async_tcp._stream_reader._loop
assert async_loop_of_connection0 == async_loop_of_connection1
def test_can_notify_its_observer_about_data_comming_from_external_io(
buffer_transport_class):
from moler.connection import ObservableConnection
moler_received_data = []
def buffer_observer(data):
moler_received_data.append(data)
moler_conn = ObservableConnection()
moler_conn.subscribe(buffer_observer)
used_io = buffer_transport_class(
moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(
input_bytes=b"incoming data") # inject to buffer for next line read
used_io.read()
assert b"incoming data" in moler_received_data
def test_connection_has_not_stopped_loop_after_close(tcp_connection_class,
integration_tcp_server_and_pipe):
from moler.connection import ObservableConnection
(tcp_server, tcp_server_pipe) = integration_tcp_server_and_pipe
moler_conn = ObservableConnection()
connection = tcp_connection_class(moler_connection=moler_conn, port=tcp_server.port, host=tcp_server.host)
connection.open()
async_loop_of_connection = connection._async_tcp._stream_reader._loop
connection.close()
assert async_loop_of_connection.is_running()
def test_can_receive_binary_data_from_connection(tcp_connection_class,
integration_tcp_server_and_pipe):
from moler.connection import ObservableConnection
(tcp_server, tcp_server_pipe) = integration_tcp_server_and_pipe
received_data = bytearray()
receiver_called = threading.Event()
def receiver(data):
received_data.extend(data)
receiver_called.set()
moler_conn = ObservableConnection() # no decoder, just pass bytes 1:1
moler_conn.subscribe(receiver) # build forwarding path
connection = tcp_connection_class(moler_connection=moler_conn, port=tcp_server.port, host=tcp_server.host)
with connection.open(): # TODO: async with connection.open():
time.sleep(0.1) # otherwise we have race between server's pipe and from-client-connection
tcp_server_pipe.send(("send async msg", {'msg': b'data to read'}))
receiver_called.wait(timeout=0.5)
assert b'data to read' == received_data
def test_notified_observer_may_stop_subscription_of_data_comming_from_external_io(buffer_transport_class):
from moler.connection import ObservableConnection
moler_conn = ObservableConnection()
moler_received_data = []
def one_time_observer(data):
moler_received_data.append(data)
moler_conn.unsubscribe(one_time_observer)
moler_conn.subscribe(one_time_observer)
used_io = buffer_transport_class(moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"data 1") # inject to buffer for next line read
used_io.read()
used_io.write(input_bytes=b"data 2") # inject to buffer for next line read
used_io.read()
assert b"data 1" in moler_received_data
assert b"data 2" not in moler_received_data # because of unsubscription during notification
def ping_observing_task(address, ping_ip):
logger = logging.getLogger('moler.user.app-code')
net_addr = 'tcp://{}:{}'.format(*address)
# Lowest layer of Moler's usage (you manually glue all elements):
# 1. create observers
net_down_detector = NetworkDownDetector(ping_ip)
net_drop_found = False
net_up_detector = NetworkUpDetector(ping_ip)
moler_conn = ObservableConnection(
decoder=lambda data: data.decode("utf-8"))
# 2. virtually "start" observer by making it data-listener
moler_conn.subscribe(net_down_detector.data_received)
info = '{} on {} using {}'.format(ping_ip, net_addr, net_down_detector)
logger.debug('observe ' + info)
for _ in tcp_connection(address, moler_conn):
# anytime new data comes it may change status of observer
if not net_drop_found and net_down_detector.done():
net_drop_found = True
net_down_time = net_down_detector.result()
timestamp = time.strftime("%H:%M:%S",
time.localtime(net_down_time))
logger.debug('Network {} is down from {}'.format(
ping_ip, timestamp))
# 3. virtually "stop" that observer
moler_conn.unsubscribe(net_down_detector.data_received)
# 4. and start subsequent one (to know when net is back "up")
info = '{} on {} using {}'.format(ping_ip, net_addr,
net_up_detector)
logger.debug('observe ' + info)
moler_conn.subscribe(net_up_detector.data_received)
if net_up_detector.done():
net_up_time = net_up_detector.result()
timestamp = time.strftime("%H:%M:%S", time.localtime(net_up_time))
logger.debug('Network {} is back "up" from {}'.format(
ping_ip, timestamp))
# 5. virtually "stop" that observer
moler_conn.unsubscribe(net_up_detector.data_received)
break
async def ping_observing_task(address):
logger = logging.getLogger('moler.user.app-code')
# Lowest layer of Moler's usage (you manually glue all elements):
# 1. create observer
net_down_detector = NetworkDownDetector()
# 2. ObservableConnection is a proxy-glue between observer (speaks str)
# and curio-connection (speaks bytes)
moler_conn = ObservableConnection(
decoder=lambda data: data.decode("utf-8"))
# 3a. glue from proxy to observer
moler_conn.subscribe(net_down_detector.data_received)
logger.debug('waiting for data to observe')
async with curio.meta.finalize(tcp_connection(address)) as tcp_conn:
async for connection_data in tcp_conn:
# 3b. glue to proxy from external-IO (curio tcp client connection)
# (client has to pass it's received data into Moler's connection)
moler_conn.data_received(connection_data)
# 4. Moler's client code must manually check status of observer ...
if net_down_detector.done():
# 5. ... to know when it can ask for result
net_down_time = net_down_detector.result()
timestamp = time.strftime("%H:%M:%S",
time.localtime(net_down_time))
logger.debug('Network is down from {}'.format(timestamp))
break
def test_repeated_unsubscription_does_nothing_but_logs_warning(
buffer_transport_class):
"""
Because of possible different concurrency models (and their races)
we don't want to raise exception when there is already
"no such subscription" - just put warning to logs
"""
from moler.connection import ObservableConnection
moler_conn = ObservableConnection()
moler_received_data = []
def one_time_observer(data):
moler_received_data.append(data)
moler_conn.unsubscribe(one_time_observer)
moler_conn.subscribe(one_time_observer)
used_io = buffer_transport_class(
moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"data 1") # inject to buffer for next line read
used_io.read()
moler_conn.unsubscribe(
one_time_observer
) # TODO: check warning in logs (when we set logging system)
used_io.write(input_bytes=b"data 2") # inject to buffer for next line read
used_io.read()
assert b"data 1" in moler_received_data
assert b"data 2" not in moler_received_data # because of unsubscription during notification
def test_events_false_any():
connection = ObservableConnection()
events = list()
patterns = ("aaa", "bbb")
for pattern in patterns:
event = Wait4prompt(connection=connection, till_occurs_times=1, prompt=pattern)
event.start()
events.append(event)
assert EventAwaiter.wait_for_any(timeout=0.1, events=events) is False
done, not_done = EventAwaiter.separate_done_events(events)
assert 0 == len(done)
assert 2 == len(not_done)
EventAwaiter.cancel_all_events(events)
def test_can_notify_multiple_observers_about_data_comming_from_external_io(buffer_transport_class):
from moler.connection import ObservableConnection
class BufferObserver(object):
def __init__(self):
self.received_data = []
def on_new_data(self, data):
self.received_data.append(data)
buffer_observer1 = BufferObserver()
buffer_observer2 = BufferObserver()
moler_conn = ObservableConnection()
moler_conn.subscribe(buffer_observer1.on_new_data)
moler_conn.subscribe(buffer_observer2.on_new_data)
used_io = buffer_transport_class(moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"incoming data") # inject to buffer for next line read
used_io.read()
assert b"incoming data" in buffer_observer1.received_data
assert b"incoming data" in buffer_observer2.received_data
def test_can_open_and_close_connection_as_context_manager(tcp_connection_class,
integration_tcp_server_and_pipe):
from moler.connection import ObservableConnection
(tcp_server, tcp_server_pipe) = integration_tcp_server_and_pipe
moler_conn = ObservableConnection()
connection = tcp_connection_class(moler_connection=moler_conn, port=tcp_server.port, host=tcp_server.host)
with connection.open():
pass
time.sleep(0.1) # otherwise we have race between server's pipe and from-client-connection
tcp_server_pipe.send(("get history", {}))
dialog_with_server = tcp_server_pipe.recv()
assert 'Client connected' in dialog_with_server
assert 'Client disconnected' in dialog_with_server
def connection_to_remote():
"""
Any external-IO connection that embeds Moler-connection
Alows to check if data send from command has reached remote side via:
`data in conn.remote_endpoint()`
"""
class RemoteConnection(FifoBuffer):
def remote_endpoint(self):
"""Simulate remote endpoint that gets data"""
return self.buffer
ext_io = RemoteConnection(moler_connection=ObservableConnection(encoder=lambda data: data.encode("utf-8"),
decoder=lambda data: data.decode("utf-8")))
return ext_io
def test_command_string_is_required_to_start_command(command_major_base_class):
from moler.exceptions import NoCommandStringProvided
moler_conn = ObservableConnection()
command_class = do_nothing_command_class(base_class=command_major_base_class)
command = command_class(connection=moler_conn)
assert not command.command_string # ensure it is empty before starting command
with pytest.raises(NoCommandStringProvided) as error:
command.start() # start the command-future (background run)
assert error.value.command == command
assert 'for {}'.format(str(command)) in str(error.value)
assert 'You should fill .command_string member before starting command' in str(error.value)
def failing_net_down_detector(fail_on_data, fail_by_raising, runner):
from moler.connection import ObservableConnection
class FailingNetworkDownDetector(NetworkDownDetector):
def data_received(self, data):
if data == fail_on_data:
raise fail_by_raising
return super(FailingNetworkDownDetector, self).data_received(data)
moler_conn = ObservableConnection()
failing_detector = FailingNetworkDownDetector(connection=moler_conn,
runner=runner)
yield failing_detector
# remove exceptions collected inside ConnectionObserver
ConnectionObserver.get_unraised_exceptions(remove=True)
def test_runner_shutdown_cancels_remaining_inactive_feeders_inside_main_thread(
observer_runner):
from moler.connection import ObservableConnection
connection_observer = NetworkDownDetector(
connection=ObservableConnection(), runner=observer_runner)
connection_observer.start_time = time.time(
) # must start observer lifetime before runner.submit()
future = observer_runner.submit(connection_observer)
time.sleep(
0.2
) # won't enter event loop of future - feeder won't start processing
observer_runner.shutdown()
assert connection_observer.cancelled()
def test_runner_shutdown_cancels_remaining_active_feeders_inside_main_thread(
async_runner):
from moler.connection import ObservableConnection
connection_observer = NetworkDownDetector(
connection=ObservableConnection(), runner=async_runner)
connection_observer.start_time = time.time(
) # must start observer lifetime before runner.submit()
future = async_runner.submit(connection_observer)
future._loop.run_until_complete(
asyncio.sleep(1.0)) # feeder will start processing inside loop
# time.sleep(0.5)
async_runner.shutdown()
assert connection_observer.cancelled()
def test_closing_closed_connection_does_nothing(tcp_connection_class,
integration_tcp_server_and_pipe):
from moler.connection import ObservableConnection
(tcp_server, tcp_server_pipe) = integration_tcp_server_and_pipe
moler_conn = ObservableConnection()
connection = tcp_connection_class(moler_connection=moler_conn, port=tcp_server.port, host=tcp_server.host)
connection.open()
connection.close()
connection.close()
time.sleep(0.1) # otherwise we have race between server's pipe and from-client-connection
tcp_server_pipe.send(("get history", {}))
dialog_with_server = tcp_server_pipe.recv()
assert 'Client connected' in dialog_with_server
assert 'Client disconnected' in dialog_with_server
assert dialog_with_server[-2] != 'Client disconnected' # not closed twice
def test_gets_all_data_of_connection_after_it_is_started(observer_runner):
from moler.connection import ObservableConnection
for n in range(20): # need to test multiple times because of thread races
moler_conn = ObservableConnection()
net_down_detector = NetworkDownDetector(connection=moler_conn,
runner=observer_runner)
connection = net_down_detector.connection
net_down_detector.start()
connection.data_received("61 bytes")
connection.data_received("62 bytes")
connection.data_received("ping: Network is unreachable")
assert net_down_detector.all_data_received == [
"61 bytes", "62 bytes", "ping: Network is unreachable"
]
async def test_wait_for__prohibited_inside_async_def_speaks_in_observer_API(
async_runner):
from moler.exceptions import WrongUsage
from moler.connection import ObservableConnection
connection_observer = NetworkDownDetector(
connection=ObservableConnection(), runner=async_runner)
connection_observer.start() # internally calls async_runner.submit()
future = async_runner.submit(connection_observer)
with pytest.raises(WrongUsage) as err:
connection_observer.await_done(
) # internally calls async_runner.wait_for() + connection_observer.result()
assert "Can't call await_done() from 'async def' - it is blocking call" in str(
err.value)
# check "fix-hint" inside exception
assert re.findall(
r'consider using:\s+await observer\s+instead of:\s+observer.await_done()',
str(err.value))
def test_can_open_and_close_connection(tcp_connection_class,
integration_tcp_server_and_pipe):
"""
Not so atomic test (checks 2 things) but:
- it is integration tests
- anyway open needs close as cleanup to not have resources leaking in tests
"""
from moler.connection import ObservableConnection
(tcp_server, tcp_server_pipe) = integration_tcp_server_and_pipe
moler_conn = ObservableConnection()
connection = tcp_connection_class(moler_connection=moler_conn, port=tcp_server.port, host=tcp_server.host)
connection.open()
connection.close()
time.sleep(0.1) # otherwise we have race between server's pipe and from-client-connection
tcp_server_pipe.send(("get history", {}))
dialog_with_server = tcp_server_pipe.recv()
assert 'Client connected' in dialog_with_server
assert 'Client disconnected' in dialog_with_server
def buffer_connection():
"""External-io based on memory FIFO-buffer"""
from moler.io.raw.memory import ThreadedFifoBuffer
from moler.connection import ObservableConnection
class RemoteConnection(ThreadedFifoBuffer):
def remote_inject_response(self, input_strings, delay=0.0):
"""
Simulate remote endpoint that sends response.
Response is given as strings.
"""
in_bytes = [data.encode("utf-8") for data in input_strings]
self.inject_response(in_bytes, delay)
moler_conn = ObservableConnection(
encoder=lambda data: data.encode("utf-8"),
decoder=lambda data: data.decode("utf-8"))
ext_io_in_memory = RemoteConnection(moler_connection=moler_conn,
echo=False) # we don't want echo on it
return ext_io_in_memory
def buffer_connection():
from moler.io.raw.memory import ThreadedFifoBuffer
from moler.connection import ObservableConnection
class RemoteConnection(ThreadedFifoBuffer):
def remote_inject_response(self, input_strings, delay=0.0):
"""
Simulate remote endpoint that sends response.
Response is given as strings.
"""
in_bytes = [data.encode("utf-8") for data in input_strings]
self.inject_response(in_bytes, delay)
moler_conn = ObservableConnection(
encoder=lambda data: data.encode("utf-8"),
decoder=lambda data: data.decode("utf-8"))
ext_io_in_memory = RemoteConnection(
moler_connection=moler_conn,
echo=False) # we don't want echo on connection
# all tests assume working with already open connection
with ext_io_in_memory: # open it (autoclose by context-mngr)
yield ext_io_in_memory
def main(connections2observe4ip):
# Starting the clients
connections = []
for address, ping_ip in connections2observe4ip:
host, port = address
# 1. create Moler's connection that knows encoding
decoder = lambda data: data.decode("utf-8")
moler_conn = ObservableConnection(decoder=decoder)
# 2. create external-IO connection gluing to Moler's connection
conn_logger_name = 'threaded.tcp-connection({}:{})'.format(*address)
conn_logger = logging.getLogger(conn_logger_name)
tcp_connection = tcp.ThreadedTcp(moler_connection=moler_conn,
port=port,
host=host,
logger=conn_logger)
client_thread = threading.Thread(target=ping_observing_task,
args=(tcp_connection, ping_ip))
client_thread.start()
connections.append(client_thread)
# await observers job to be done
for client_thread in connections:
client_thread.join()
def test_exception_in_observer_doesnt_break_connection_nor_other_observers(
buffer_transport_class):
from moler.connection import ObservableConnection
moler_conn = ObservableConnection()
moler_received_data = []
def failing_observer(data):
raise Exception("Fail inside observer")
def one_time_observer(data):
moler_received_data.append(data)
moler_conn.unsubscribe(one_time_observer)
moler_conn.subscribe(failing_observer)
moler_conn.subscribe(one_time_observer)
used_io = buffer_transport_class(
moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"data 1") # inject to buffer for next line read
used_io.read()
moler_conn.unsubscribe(failing_observer)
assert b"data 1" in moler_received_data
async def test_runner_unsubscribes_from_connection_after_runner_shutdown(
observer_runner):
# see - Raw 'def' usage note
from moler.connection import ObservableConnection
moler_conn = ObservableConnection()
# check if shutdown unsubscribes all observers running inside given runner
net_down_detector1 = NetworkDownDetector(connection=moler_conn,
runner=observer_runner)
net_down_detector2 = NetworkDownDetector(connection=moler_conn,
runner=observer_runner)
net_down_detector1.start_time = time.time(
) # must start observer lifetime before runner.submit()
net_down_detector2.start_time = time.time(
) # must start observer lifetime before runner.submit()
assert len(moler_conn._observers) == 0
observer_runner.submit(net_down_detector1)
observer_runner.submit(net_down_detector2)
assert len(moler_conn._observers) == 2
observer_runner.shutdown()
await asyncio.sleep(0.1)
assert len(moler_conn._observers) == 0
def test_garbage_collected_subscriber_is_not_notified():
from moler.connection import ObservableConnection
moler_conn = ObservableConnection()
received_data = []
class Subscriber(object):
def __call__(self, data):
received_data.append(data)
subscr1 = Subscriber()
subscr2 = Subscriber()
moler_conn.subscribe(subscr1)
moler_conn.subscribe(subscr2)
del subscr1
gc.collect()
moler_conn.data_received("data")
assert len(received_data) == 1
def connection_observer():
from moler.connection import ObservableConnection
moler_conn = ObservableConnection()
observer = NetworkDownDetector(connection=moler_conn)
return observer