def test_can_receive_binary_data_from_connection(
tcp_connection_class, integration_tcp_server_and_pipe):
from moler.observable_connection import ObservableConnection
(tcp_server, tcp_server_pipe) = integration_tcp_server_and_pipe
received_data = bytearray()
receiver_called = threading.Event()
def receiver(data):
received_data.extend(data)
receiver_called.set()
def connection_closed_handler():
pass
moler_conn = ObservableConnection() # no decoder, just pass bytes 1:1
moler_conn.subscribe(receiver,
connection_closed_handler) # build forwarding path
connection = tcp_connection_class(moler_connection=moler_conn,
port=tcp_server.port,
host=tcp_server.host)
with connection.open():
time.sleep(
0.1
) # otherwise we have race between server's pipe and from-client-connection
tcp_server_pipe.send(("send async msg", {'msg': b'data to read'}))
receiver_called.wait(timeout=0.5)
assert b'data to read' == received_data
async def terminal_io_test():
from moler.observable_connection import ObservableConnection
received_data = []
moler_conn = ObservableConnection(encoder=lambda data: data.encode("utf-8"),
decoder=lambda data: data.decode("utf-8"))
terminal = AsyncioTerminal(moler_connection=moler_conn)
cmds = ['pwd', 'ssh [email protected]', 'password', 'ls\r', 'exit\r']
cmd_idx = [0]
def data_observer(data):
print(data)
received_data.append(data)
print(received_data)
if cmd_idx[0] < len(cmds):
cmd2send = cmds[cmd_idx[0]]
if (cmd2send == 'password') and ('Password' not in data):
return
moler_conn.send(data=cmd2send + '\n')
cmd_idx[0] += 1
moler_conn.subscribe(data_observer)
await terminal.open()
await asyncio.sleep(10)
await terminal.close()
print("end of test")
def ping_observing_task(address):
logger = logging.getLogger('moler.user.app-code')
observer_done = Deferred()
# Lowest layer of Moler's usage (you manually glue all elements):
# 1. create observer
net_down_detector = NetworkDownDetector('10.0.2.15')
# 2. ObservableConnection is a proxy-glue between observer (speaks str)
# and twisted-connection (speaks bytes)
moler_conn = ObservableConnection(
decoder=lambda data: data.decode("utf-8"))
# 3a. glue from proxy to observer
moler_conn.subscribe(net_down_detector.data_received)
logger.debug('waiting for data to observe')
def feed_moler(connection_data):
# 3b. glue to proxy from external-IO (twisted tcp client connection)
# (client has to pass it's received data into Moler's connection)
moler_conn.data_received(connection_data)
# 4. Moler's client code must manually check status of observer ...
if net_down_detector.done():
# 5. ... to know when it can ask for result
net_down_time = net_down_detector.result()
timestamp = time.strftime("%H:%M:%S",
time.localtime(net_down_time))
logger.debug('Network is down from {}'.format(timestamp))
observer_done.callback(None) # break tcp client and server
start_tcp_connection(address, feed_moler)
return observer_done
def test_exception_in_observer_doesnt_break_connection_nor_other_observers(
buffer_transport_class):
from moler.observable_connection import ObservableConnection
moler_conn = ObservableConnection()
moler_received_data = []
def failing_observer(data):
raise Exception("Fail inside observer")
def one_time_observer(data):
moler_received_data.append(data)
moler_conn.unsubscribe(observer=one_time_observer,
connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=failing_observer,
connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=one_time_observer,
connection_closed_handler=do_nothing_func)
used_io = buffer_transport_class(
moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"data 1") # inject to buffer for next line read
used_io.read()
moler_conn.unsubscribe(observer=failing_observer,
connection_closed_handler=do_nothing_func)
assert b"data 1" in moler_received_data
def connection_observer(observer_runner):
from moler.observable_connection import ObservableConnection
moler_conn = ObservableConnection()
observer = NetworkDownDetector(connection=moler_conn, runner=observer_runner)
yield observer
# remove exceptions collected inside ConnectionObserver
ConnectionObserver.get_unraised_exceptions(remove=True)
def test_runner_secures_observer_against_additional_data_after_observer_is_done(observer_runner):
"""Done observer should not get data even before unsubscribe from moler-connection"""
# correctly written observer looks like:
#
# def data_received(self, data):
# if not self.done():
# parse(data)
#
# This test checks if runners secure wrong-written-observers with missing 'if not self.done():'
from moler.observable_connection import ObservableConnection
with disabled_logging():
for n in range(20): # need to test multiple times to ensure there are no thread races
moler_conn = ObservableConnection()
net_down_detector = NetworkDownDetector(connection=moler_conn, runner=observer_runner)
net_down_detector.start_time = time.time() # must start observer lifetime before runner.submit()
connection = net_down_detector.connection
net_down_detector.start_time = time.time()
observer_runner.submit(net_down_detector)
connection.data_received("61 bytes")
connection.data_received("ping: Network is unreachable")
connection.data_received("62 bytes")
assert net_down_detector.all_data_received == ["61 bytes", "ping: Network is unreachable"]
def test_runner_shutdown_cancels_remaining_feeders_inside_threads(observer_runner):
from moler.observable_connection import ObservableConnection
observers_pool = []
for idx in range(3):
connection_observer = NetworkDownDetector(connection=ObservableConnection(), runner=observer_runner)
observers_pool.append(connection_observer)
def submit_feeder(connection_observer):
connection_observer.start_time = time.time() # must start observer lifetime before runner.submit()
future = observer_runner.submit(connection_observer)
while not future.done():
time.sleep(0.1)
th_pool = [threading.Thread(target=submit_feeder, args=(connection_observer,)) for connection_observer in observers_pool]
for th in th_pool:
th.start()
# loop.run_until_complete(remaining_tasks) # let it enter feeder
time.sleep(0.5)
observer_runner.shutdown()
for th in th_pool:
th.join()
assert observers_pool[0].cancelled()
assert observers_pool[1].cancelled()
assert observers_pool[2].cancelled()
def test_repr_conversion_of_command_object():
"""
repr() conversion shows same as str() plus embedded connection used by command
"""
moler_conn = ObservableConnection(
decoder=lambda data: data.decode("utf-8"))
class LsCmd(Command):
def __init__(self, options='-l', connection=None):
super(LsCmd, self).__init__(connection=connection)
self.command_string = 'ls {}'.format(options)
def data_received(self, data):
pass # not important now
ls = LsCmd(connection=moler_conn)
# (1) command with ObservableConnection to glued to ext-io
assert 'LsCmd("ls -l", id:{}, using ObservableConnection(id:{})-->[?])'.format(
instance_id(ls), instance_id(moler_conn)) == repr(ls)
# TODO: add test for <ObservableConnection( id:{}>
# (2) command with ObservableConnection glued to ext-io
ext_io_connection = FifoBuffer(moler_connection=moler_conn)
how2send_repr = repr(ext_io_connection.write)
assert 'LsCmd("ls -l", id:{}, using ObservableConnection(id:{})-->[{}])'.format(
instance_id(ls), instance_id(moler_conn), how2send_repr) == repr(ls)
# TODO: move ObservableConnection(id:{})-->[{}])'.format(instance_id(moler_conn), how2send_repr) into ObservableConnection __repr__ test
# TODO: and here just:
# assert 'LsCmd("ls -l", id:{}, using {})'.format(instance_id(ls), repr(moler_conn)) == repr(ls)
# (3) command without connection
ls.connection = None
assert 'LsCmd("ls -l", id:{}, using <NO CONNECTION>)'.format(
instance_id(ls)) == repr(ls)
async def test_observer_gets_all_data_of_connection_after_it_is_submitted_to_background(observer_runner):
# another words: after returning from runner.submit() no data can be lost, no races
# Raw 'def' usage note:
# This functionality works as well when runner is used inside raw def function
# since it only uses runner.submit() + awaiting time
# another words - runner is running over some time period
# The only difference is that raw def function may use only standalone_runner (which is subset of observer_runner)
# and inside test you exchange 'await asyncio.sleep()' with 'time.sleep()'
from moler.observable_connection import ObservableConnection
with disabled_logging():
durations = []
for n in range(20): # need to test multiple times to ensure there are no thread races
moler_conn = ObservableConnection()
net_down_detector = NetworkDownDetector(connection=moler_conn, runner=observer_runner)
connection = net_down_detector.connection
start_time = net_down_detector.start_time = time.time()
observer_runner.submit(net_down_detector)
durations.append(time.time() - start_time)
connection.data_received("61 bytes")
connection.data_received("62 bytes")
connection.data_received("ping: Network is unreachable")
assert net_down_detector.all_data_received == ["61 bytes", "62 bytes", "ping: Network is unreachable"]
print("\n{}.submit() duration == {}".format(observer_runner.__class__.__name__,
float(sum(durations))/len(durations)))
def buffer_connection():
from moler.io.raw.memory import ThreadedFifoBuffer
from moler.observable_connection import ObservableConnection
from moler.config.loggers import configure_device_logger
class RemoteConnection(ThreadedFifoBuffer):
def remote_inject_response(self, input_strings, delay=0.0):
"""
Simulate remote endpoint that sends response.
Response is given as strings.
"""
in_bytes = [data.encode("utf-8") for data in input_strings]
self.inject_response(in_bytes, delay)
moler_conn = ObservableConnection(
encoder=lambda data: data.encode("utf-8"),
decoder=lambda data: data.decode("utf-8"),
name="buffer")
ext_io_in_memory = RemoteConnection(
moler_connection=moler_conn,
echo=False) # we don't want echo on connection
configure_device_logger(moler_conn.name)
# all tests assume working with already open connection
with ext_io_in_memory.open(): # open it (autoclose by context-mngr)
yield ext_io_in_memory
def test_events_true_any_one():
connection = ObservableConnection()
events = list()
patterns = ("aaa", "bbb")
for pattern in patterns:
event = Wait4prompt(connection=connection,
till_occurs_times=1,
prompt=pattern)
event.start()
events.append(event)
connection.data_received(patterns[0])
assert EventAwaiter.wait_for_any(timeout=0.1, events=events) is True
done, not_done = EventAwaiter.separate_done_events(events)
assert 1 == len(done)
assert 1 == len(not_done)
EventAwaiter.cancel_all_events(events)
def ext_io_connection(request):
from moler.io.raw.memory import ThreadedFifoBuffer
from moler.observable_connection import ObservableConnection
moler_conn = ObservableConnection(
decoder=lambda data: data.decode("utf-8"))
connection = ThreadedFifoBuffer(moler_connection=moler_conn)
return connection
def test_command_string_is_required_to_call_command(command_major_base_class):
import threading
from moler.exceptions import NoCommandStringProvided
moler_conn = ObservableConnection()
command_class = do_nothing_command_class(
base_class=command_major_base_class)
command = command_class(connection=moler_conn)
assert not command.command_string # ensure it is empty before starting command
def command_in_thread():
with pytest.raises(NoCommandStringProvided) as error:
command()
assert error.value.command == command
assert 'for {}'.format(str(command)) in str(error.value)
assert 'You should fill .command_string member before starting command' in str(
error.value)
cmd_thrd = threading.Thread(target=command_in_thread)
cmd_thrd.start()
cmd_thrd.join()
command = command_class(connection=moler_conn)
with pytest.raises(NoCommandStringProvided) as error:
command() # call the command-future (foreground run)
assert error.value.command == command
assert 'for {}'.format(str(command)) in str(error.value)
assert 'You should fill .command_string member before starting command' in str(
error.value)
def memory_connection_without_decoder(memory_connection_class):
connection_class = memory_connection_class
from moler.observable_connection import ObservableConnection
moler_conn = ObservableConnection(
encoder=lambda data: data.encode("utf-8"))
connection = connection_class(moler_connection=moler_conn)
return connection
def test_can_send_binary_data_over_connection(tcp_connection_class,
integration_tcp_server_and_pipe):
from moler.observable_connection import ObservableConnection
(tcp_server, tcp_server_pipe) = integration_tcp_server_and_pipe
moler_conn = ObservableConnection() # no decoder, just pass bytes 1:1
connection = tcp_connection_class(moler_connection=moler_conn,
port=tcp_server.port,
host=tcp_server.host)
with connection.open():
moler_conn.send(data=b'data to be send')
time.sleep(
0.1
) # otherwise we have race between server's pipe and from-client-connection
tcp_server_pipe.send(("get history", {}))
dialog_with_server = tcp_server_pipe.recv()
assert ['Received data:', b'data to be send'] == dialog_with_server[-1]
def terminal_connection():
from moler.observable_connection import ObservableConnection
moler_conn = ObservableConnection()
terminal = ThreadedTerminal(moler_connection=moler_conn)
with terminal.open() as connection:
yield connection.moler_connection
def conn_observer(request):
moler_conn = ObservableConnection(how2send=mock.MagicMock())
if request.param == 'generic_observer':
observer = NetworkDownDetector(connection=moler_conn)
elif request.param == 'event':
observer = MyEvent(connection=moler_conn)
elif request.param == 'command':
observer = MyCommand(connection=moler_conn)
return observer
def test_event_string_is_required_to_start_command(lineevent_class):
from moler.exceptions import NoDetectPatternProvided
moler_conn = ObservableConnection()
event_class = do_nothing_command_class(base_class=lineevent_class)
event = event_class(connection=moler_conn, detect_patterns=[])
assert not event.detect_patterns # ensure it is empty before starting command
with pytest.raises(NoDetectPatternProvided) as error:
event.start() # start the command-future (background run)
def tcp_thd_conn(port, host='localhost', name=None):
moler_conn = ObservableConnection(
decoder=lambda data: data.decode("utf-8"))
conn_logger_name = 'threaded.tcp-connection({}:{})'.format(host, port)
conn_logger = logging.getLogger(conn_logger_name)
io_conn = tcp.ThreadedTcp(moler_connection=moler_conn,
port=port,
host=host,
logger=conn_logger)
return io_conn
def test_runner_shutdown_cancels_remaining_inactive_feeders_inside_main_thread(observer_runner):
from moler.observable_connection import ObservableConnection
connection_observer = NetworkDownDetector(connection=ObservableConnection(), runner=observer_runner)
connection_observer.start_time = time.time() # must start observer lifetime before runner.submit()
future = observer_runner.submit(connection_observer)
time.sleep(0.2) # won't enter event loop of future - feeder won't start processing
observer_runner.shutdown()
assert connection_observer.cancelled()
def test_can_notify_its_observer_about_data_comming_from_external_io(
buffer_transport_class):
from moler.observable_connection import ObservableConnection
moler_received_data = []
def buffer_observer(data):
moler_received_data.append(data)
moler_conn = ObservableConnection()
moler_conn.subscribe(observer=buffer_observer,
connection_closed_handler=do_nothing_func)
used_io = buffer_transport_class(
moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(
input_bytes=b"incoming data") # inject to buffer for next line read
used_io.read()
assert b"incoming data" in moler_received_data
def test_subscription_doesnt_block_subscriber_to_be_garbage_collected():
from moler.observable_connection import ObservableConnection
moler_conn = ObservableConnection()
garbage_collected_subscribers = []
class Subscriber(object):
def __del__(self):
garbage_collected_subscribers.append('Subscriber')
class CloseSubscriber(object):
pass
subscr = Subscriber()
close_subscr = CloseSubscriber()
moler_conn.subscribe(subscr, close_subscr)
del subscr
gc.collect()
assert 'Subscriber' in garbage_collected_subscribers
def test_runner_shutdown_cancels_remaining_active_feeders_inside_main_thread(async_runner):
from moler.observable_connection import ObservableConnection
connection_observer = NetworkDownDetector(connection=ObservableConnection(), runner=async_runner)
connection_observer.start_time = time.time() # must start observer lifetime before runner.submit()
future = async_runner.submit(connection_observer)
future._loop.run_until_complete(asyncio.sleep(1.0)) # feeder will start processing inside loop
# time.sleep(0.5)
async_runner.shutdown()
assert connection_observer.cancelled()
def test_repeated_unsubscription_does_nothing_but_logs_warning(
buffer_transport_class):
"""
Because of possible different concurrency models (and their races)
we don't want to raise exception when there is already
"no such subscription" - just put warning to logs
"""
from moler.observable_connection import ObservableConnection
moler_conn = ObservableConnection()
moler_received_data = []
def one_time_observer(data):
moler_received_data.append(data)
moler_conn.unsubscribe(observer=one_time_observer,
connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=one_time_observer,
connection_closed_handler=do_nothing_func)
used_io = buffer_transport_class(
moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"data 1") # inject to buffer for next line read
used_io.read()
moler_conn.unsubscribe(
observer=one_time_observer,
connection_closed_handler=do_nothing_func) # TODO: check
# warning in logs (when we set logging system)
used_io.write(input_bytes=b"data 2") # inject to buffer for next line read
used_io.read()
assert b"data 1" in moler_received_data
assert b"data 2" not in moler_received_data # because of unsubscription during notification
def test_notifies_only_subscribed_observers_about_data_comming_from_external_io(
buffer_transport_class):
from moler.observable_connection import ObservableConnection
class BufferObserver(object):
def __init__(self):
self.received_data = []
def on_new_data(self, data):
self.received_data.append(data)
buffer_observer1 = BufferObserver()
buffer_observer2 = BufferObserver()
buffer_observer3 = BufferObserver()
moler_conn = ObservableConnection()
moler_conn.subscribe(observer=buffer_observer1.on_new_data,
connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=buffer_observer2.on_new_data,
connection_closed_handler=do_nothing_func)
used_io = buffer_transport_class(
moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(
input_bytes=b"incoming data") # inject to buffer for next line read
used_io.read()
assert b"incoming data" in buffer_observer1.received_data
assert b"incoming data" in buffer_observer2.received_data
assert b"incoming data" not in buffer_observer3.received_data # that one was not subscribed
def ping_observing_task(address, ping_ip):
logger = logging.getLogger('moler.user.app-code')
net_addr = 'tcp://{}:{}'.format(*address)
# Lowest layer of Moler's usage (you manually glue all elements):
# 1. create observers
net_down_detector = NetworkDownDetector(ping_ip)
net_drop_found = False
net_up_detector = NetworkUpDetector(ping_ip)
moler_conn = ObservableConnection(
decoder=lambda data: data.decode("utf-8"))
# 2. virtually "start" observer by making it data-listener
moler_conn.subscribe(net_down_detector.data_received)
info = '{} on {} using {}'.format(ping_ip, net_addr, net_down_detector)
logger.debug('observe ' + info)
for _ in tcp_connection(address, moler_conn):
# anytime new data comes it may change status of observer
if not net_drop_found and net_down_detector.done():
net_drop_found = True
net_down_time = net_down_detector.result()
timestamp = time.strftime("%H:%M:%S",
time.localtime(net_down_time))
logger.debug('Network {} is down from {}'.format(
ping_ip, timestamp))
# 3. virtually "stop" that observer
moler_conn.unsubscribe(net_down_detector.data_received)
# 4. and start subsequent one (to know when net is back "up")
info = '{} on {} using {}'.format(ping_ip, net_addr,
net_up_detector)
logger.debug('observe ' + info)
moler_conn.subscribe(net_up_detector.data_received)
if net_up_detector.done():
net_up_time = net_up_detector.result()
timestamp = time.strftime("%H:%M:%S", time.localtime(net_up_time))
logger.debug('Network {} is back "up" from {}'.format(
ping_ip, timestamp))
# 5. virtually "stop" that observer
moler_conn.unsubscribe(net_up_detector.data_received)
break
async def test_wait_for__prohibited_inside_async_def_speaks_in_observer_API(async_runner):
from moler.exceptions import WrongUsage
from moler.observable_connection import ObservableConnection
connection_observer = NetworkDownDetector(connection=ObservableConnection(), runner=async_runner)
connection_observer.start() # internally calls async_runner.submit()
future = async_runner.submit(connection_observer)
with pytest.raises(WrongUsage) as err:
connection_observer.await_done() # internally calls async_runner.wait_for() + connection_observer.result()
assert "Can't call await_done() from 'async def' - it is blocking call" in str(err.value)
# check "fix-hint" inside exception
assert re.findall(r'consider using:\s+await observer\s+instead of:\s+observer.await_done()', str(err.value))
def failing_net_down_detector(fail_on_data, fail_by_raising, runner):
from moler.observable_connection import ObservableConnection
class FailingNetworkDownDetector(NetworkDownDetector):
def data_received(self, data):
if data == fail_on_data:
raise fail_by_raising
return super(FailingNetworkDownDetector, self).data_received(data)
moler_conn = ObservableConnection()
failing_detector = FailingNetworkDownDetector(connection=moler_conn, runner=runner)
yield failing_detector
# remove exceptions collected inside ConnectionObserver
ConnectionObserver.get_unraised_exceptions(remove=True)
def connection_to_remote():
"""
Any external-IO connection that embeds Moler-connection
Alows to check if data send from command has reached remote side via:
`data in conn.remote_endpoint()`
"""
class RemoteConnection(FifoBuffer):
def remote_endpoint(self):
"""Simulate remote endpoint that gets data"""
return self.buffer
ext_io = RemoteConnection(moler_connection=ObservableConnection(
encoder=lambda data: data.encode("utf-8"),
decoder=lambda data: data.decode("utf-8")))
return ext_io
def test_notified_observer_may_stop_subscription_of_data_comming_from_external_io(
buffer_transport_class):
from moler.observable_connection import ObservableConnection
moler_conn = ObservableConnection()
moler_received_data = []
def one_time_observer(data):
moler_received_data.append(data)
moler_conn.unsubscribe(observer=one_time_observer,
connection_closed_handler=do_nothing_func)
moler_conn.subscribe(observer=one_time_observer,
connection_closed_handler=do_nothing_func)
used_io = buffer_transport_class(
moler_connection=moler_conn) # external-IO internally sets .how2send
used_io.write(input_bytes=b"data 1") # inject to buffer for next line read
used_io.read()
used_io.write(input_bytes=b"data 2") # inject to buffer for next line read
used_io.read()
assert b"data 1" in moler_received_data
assert b"data 2" not in moler_received_data # because of unsubscription during notification