def test_sub_channel_with_exception(self):
""" Whether Sub Channel exception handling is working """
chan = BaseChannel(name="test_channel4", loop=self.loop)
n1 = TestNode(name="main")
n2 = TestNode(name="sub")
n3 = ExceptNode(name="sub2")
msg = generate_msg()
chan.add(n1)
sub = chan.fork(name="Hello")
sub.add(n2, n3)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.assertEqual(n1.processed, 1, "Sub Channel not working")
with self.assertRaises(TestException) as cm:
self.clean_loop()
self.assertEqual(n2.processed, 1, "Sub Channel not working")
def test_loop_slow2(self):
""" slow tasks delay can be configured """
import pypeman.debug
pypeman.debug.enable_slow_log_stats()
self.setup_evt_loop(slow_cb_duration=0.05)
stats = pypeman.debug.stats
# logger = self.logger # get default test logger
# handler = self.loghandler # get test log handler
tst_logger = logging.getLogger('tests.debug.main_loop.slow')
print(tst_logger.handlers)
loop = self.loop
chan = BaseChannel(name="test_loop_slow2", loop=loop)
n1 = SimpleTestNode(delay=0.03, async_delay=0, logger=tst_logger, loop=loop)
n2 = SimpleTestNode(delay=0.06, logger=tst_logger, loop=loop)
chan.add(n1)
chan.add(n2)
msg = generate_msg()
# Launch channel processing
self.loop.run_until_complete(chan.start())
self.loop.run_until_complete(chan.handle(msg))
# handler.show_entries()
self.assertEqual(len(stats), 1, "should have 1 slow tasks, not %d" % len(stats))
pypeman.debug.show_slow_log_stats()
self.loop.close()
def test_sub_channel(self):
""" Whether Sub Channel is working """
chan = BaseChannel(name="test_channel3", loop=self.loop)
n1 = TestNode(name="main")
n2 = TestNode(name="sub")
n3 = TestNode(name="sub1")
n4 = TestNode(name="sub2")
msg = generate_msg()
same_chan = chan.append(n1)
self.assertEqual(chan, same_chan, "Append don't return channel.")
sub = chan.fork(name="subchannel")
sub.append(n2, n3, n4)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.assertTrue(n2.processed, "Sub Channel not working")
self.assertTrue(n3.processed, "Sub Channel not working")
self.assertTrue(n4.processed, "Sub Channel not working")
self.assertEqual(sub.name, "test_channel3.subchannel", "Subchannel name is incorrect")
def test_channel_events(self):
""" Whether BaseChannel handling return a good result """
chan = BaseChannel(name="test_channel7.5", loop=self.loop)
msg = generate_msg()
chan.add(nodes.JsonToPython(), nodes.PythonToJson())
state_sequence = [chan.status]
@events.channel_change_state.receiver
def handle_change_state(channel=None, old_state=None, new_state=None):
print(channel.name, old_state, new_state)
state_sequence.append(new_state)
@events.channel_change_state.receiver
async def handle_change_state_async(channel=None, old_state=None, new_state=None):
print(channel.name, old_state, new_state)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.loop.run_until_complete(chan.stop())
print(state_sequence)
valid_sequence = [BaseChannel.STOPPED, BaseChannel.STARTING, BaseChannel.WAITING,
BaseChannel.PROCESSING, BaseChannel.WAITING, BaseChannel.STOPPING, BaseChannel.STOPPED]
self.assertEqual(state_sequence, valid_sequence, "Sequence state is not valid")
def test_no_node_base_channel(self):
""" Whether BaseChannel handling is working even if there is no node """
chan = BaseChannel(name="test_channel2", loop=self.loop)
msg = generate_msg()
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
def test_channel_exception(self):
""" Whether BaseChannel handling return an exception in case of error in main branch """
chan = BaseChannel(name="test_channel8", loop=self.loop)
msg = generate_msg()
chan.add(nodes.JsonToPython(), nodes.PythonToJson(), ExceptNode())
# Launch channel processing
self.start_channels()
with self.assertRaises(TestException) as cm:
self.loop.run_until_complete(chan.process(msg))
def test_channel_result(self):
""" Whether BaseChannel handling return a good result """
chan = BaseChannel(name="test_channel7", loop=self.loop)
msg = generate_msg()
chan.add(nodes.JsonToPython(), nodes.PythonToJson())
# Launch channel processing
self.start_channels()
result = self.loop.run_until_complete(chan.handle(msg))
self.assertEqual(result.payload, msg.payload, "Channel handle not working")
def test_channel_subchannel(self):
""" Whether BaseChannel subchannel works """
chan = BaseChannel(name="test_channel6.5", loop=self.loop)
chan_fork = chan.fork()
chan_when = chan_fork.when(lambda: True)
chan_case1, chan_case2 = chan_when.case(lambda: True, lambda: False)
print(chan.subchannels())
self.assertEqual(len(chan.subchannels()[0]['subchannels'][0]['subchannels']), 2, "Subchannel graph not working")
def test_base_channel(self):
""" Whether BaseChannel handling is working """
chan = BaseChannel(name="test_channel1", loop=self.loop)
n = TestNode()
msg = generate_msg()
chan.add(n)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.assertTrue(n.processed, "Channel handle not working")
def test_cond_channel(self):
""" Whether Conditionnal channel is working """
chan = BaseChannel(name="test_channel5", loop=self.loop)
n1 = TestNode(name="main")
n2 = TestNode(name="end_main")
not_processed = TestNode(name="cond_notproc")
processed = TestNode(name="cond_proc")
msg = generate_msg()
chan.add(n1)
# Nodes in this channel should not be processed
cond1 = chan.when(lambda x: False, name="Toto")
# Nodes in this channel should be processed
cond2 = chan.when(True, name="condchannel")
chan.add(n2)
cond1.add(not_processed)
cond2.add(processed)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.assertFalse(not_processed.processed, "Cond Channel when condition == False not working")
self.assertTrue(processed.processed, "Cond Channel when condition == True not working")
self.assertFalse(n2.processed, "Cond Channel don't became the main path")
self.assertEqual(cond2.name, "test_channel5.condchannel", "Condchannel name is incorrect")
def test_channel_with_generator(self):
""" Whether BaseChannel with generator is working """
chan = BaseChannel(name="test_channel7.3", loop=self.loop)
chan2 = BaseChannel(name="test_channel7.31", loop=self.loop)
msg = generate_msg()
msg2 = msg.copy()
class TestIter(nodes.BaseNode):
def process(self, msg):
def iter():
for i in range(3):
yield msg
return iter()
final_node = nodes.Log()
mid_node = nodes.Log()
chan.add(TestIter(name="testiterr"), nodes.Log(), TestIter(name="testiterr"), final_node)
chan2.add(TestIter(name="testiterr"), mid_node, nodes.Drop())
# Launch channel processing
self.start_channels()
result = self.loop.run_until_complete(chan.handle(msg))
self.assertEqual(result.payload, msg.payload, "Generator node not working")
self.assertEqual(final_node.processed, 9, "Generator node not working")
result = self.loop.run_until_complete(chan2.handle(msg2))
self.assertEqual(mid_node.processed, 3, "Generator node not working with drop_node")
def test_null_message_store(self):
""" We can store a message in NullMessageStore """
chan = BaseChannel(name="test_channel9", loop=self.loop, message_store_factory=msgstore.FakeMessageStoreFactory())
n = TestNode()
msg = generate_msg(with_context=True)
chan.add(n)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.assertTrue(n.processed, "Channel handle not working")
def test_file_message_store(self):
""" We can store a message in FileMessageStore """
tempdir = tempfile.mkdtemp()
store_factory = msgstore.FileMessageStoreFactory(path=tempdir)
chan = BaseChannel(name="test_channel11", loop=self.loop, message_store_factory=store_factory)
n = TestNode()
n_error = TestConditionalErrorNode()
msg = generate_msg()
msg2 = generate_msg(timestamp=(1982, 11, 27, 12, 35))
msg3 = generate_msg(timestamp=(1982, 11, 28, 12, 35))
msg4 = generate_msg(timestamp=(1982, 11, 28, 14, 35))
# This message should be in error
msg5 = generate_msg(timestamp=(1982, 11, 12, 14, 35))
chan.add(n)
chan.add(n_error)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.loop.run_until_complete(chan.handle(msg2))
self.loop.run_until_complete(chan.handle(msg3))
self.loop.run_until_complete(chan.handle(msg4))
with self.assertRaises(TestException) as cm:
# This message should be in error state
self.loop.run_until_complete(chan.handle(msg5))
msg_stored = list(chan.message_store.search())
for msg in msg_stored:
print(msg)
# All message stored ?
self.assertEqual(len(msg_stored), 5, "Should be 5 messages in store!")
# Test processed message
dict_msg = chan.message_store.get('1982/11/28/19821128_1235_%s' % msg3.uuid.hex)
self.assertEqual(dict_msg['state'], 'processed', "Message %s should be in processed state!" % msg3)
# Test failed message
dict_msg = chan.message_store.get('1982/11/12/19821112_1435_%s' % msg5.uuid.hex)
self.assertEqual(dict_msg['state'], 'error', "Message %s should be in error state!" % msg5)
self.assertTrue(os.path.exists("%s/%s/1982/11/28/19821128_1235_%s" % (tempdir, chan.name, msg3.uuid.hex)))
# TODO put in tear_down ?
shutil.rmtree(tempdir, ignore_errors=True)
def test_case_channel(self):
""" Whether Conditionnal channel is working """
chan = BaseChannel(name="test_channel6", loop=self.loop)
n1 = TestNode(name="main")
n2 = TestNode(name="end_main")
not_processed = TestNode(name="cond_notproc")
processed = TestNode(name="cond_proc")
not_processed2 = TestNode(name="cond_proc2")
msg = generate_msg()
chan.add(n1)
cond1, cond2, cond3 = chan.case(lambda x: False, True, True, names=['first', 'second', 'third'])
chan.add(n2)
cond1.add(not_processed)
cond2.add(processed)
cond3.add(not_processed2)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.assertFalse(not_processed.processed, "Case Channel when condition == False not working")
self.assertFalse(not_processed2.processed, "Case Channel when condition == False not working")
self.assertTrue(processed.processed, "Case Channel when condition == True not working")
self.assertTrue(n2.processed, "Cond Channel don't became the main path")
self.assertEqual(cond1.name, "test_channel6.first", "Casechannel name is incorrect")
self.assertEqual(cond2.name, "test_channel6.second", "Casechannel name is incorrect")
self.assertEqual(cond3.name, "test_channel6.third", "Casechannel name is incorrect")
def test_memory_message_store(self):
""" We can store a message in FileMessageStore """
store_factory = msgstore.MemoryMessageStoreFactory()
chan = BaseChannel(name="test_channel10", loop=self.loop, message_store_factory=store_factory)
n = TestNode()
n_error = TestConditionalErrorNode()
msg = generate_msg()
msg2 = generate_msg(timestamp=(1982, 11, 27, 12, 35))
msg3 = generate_msg(timestamp=(1982, 11, 28, 12, 35))
msg4 = generate_msg(timestamp=(1982, 11, 28, 14, 35))
# This message should be in error
msg5 = generate_msg(timestamp=(1982, 11, 12, 14, 35))
chan.add(n)
chan.add(n_error)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.loop.run_until_complete(chan.handle(msg2))
self.loop.run_until_complete(chan.handle(msg3))
self.loop.run_until_complete(chan.handle(msg4))
with self.assertRaises(TestException) as cm:
# This message should be in error state
self.loop.run_until_complete(chan.handle(msg5))
msg_stored = list(chan.message_store.search())
for msg in msg_stored:
print(msg)
# All message stored ?
self.assertEqual(len(msg_stored), 5, "Should be 5 messages in store!")
# Test processed message
dict_msg = chan.message_store.get('%s' % msg3.uuid.hex)
self.assertEqual(dict_msg['state'], 'processed', "Message %s should be in processed state!" % msg3)
# Test failed message
dict_msg = chan.message_store.get('%s' % msg5.uuid.hex)
self.assertEqual(dict_msg['state'], 'error', "Message %s should be in error state!" % msg5)
def test_channel_stopped_dont_process_message(self):
""" Whether BaseChannel handling return a good result """
chan = BaseChannel(name="test_channel7.7", loop=self.loop)
msg = generate_msg()
chan.add(nodes.JsonToPython(), nodes.PythonToJson())
@asyncio.coroutine
def go():
result = yield from chan.handle(msg)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(go())
self.loop.run_until_complete(chan.stop())
with self.assertRaises(channels.ChannelStopped) as cm:
self.loop.run_until_complete(go())
def test_replay_from_memory_message_store(self):
""" We can store a message in FileMessageStore """
store_factory = msgstore.MemoryMessageStoreFactory()
chan = BaseChannel(name="test_channel10.5", loop=self.loop, message_store_factory=store_factory)
n = TestNode()
msg = generate_msg(with_context=True)
msg2 = generate_msg(timestamp=(1982, 11, 27, 12, 35))
chan.add(n)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.loop.run_until_complete(chan.handle(msg2))
msg_stored = list(self.loop.run_until_complete(chan.message_store.search()))
for msg in msg_stored:
print(msg)
self.assertEqual(len(msg_stored), 2, "Should be 2 messages in store!")
self.loop.run_until_complete(chan.replay(msg_stored[0]['id']))
msg_stored = list(self.loop.run_until_complete(chan.message_store.search()))
self.assertEqual(len(msg_stored), 3, "Should be 3 messages in store!")
def test_memory_message_store_in_fork(self):
""" We can store a message in FileMessageStore """
store_factory = msgstore.MemoryMessageStoreFactory()
chan = BaseChannel(name="test_channel10.25", loop=self.loop, message_store_factory=store_factory)
n1 = TestNode()
n2 = TestNode()
n3 = TestNode()
n4 = TestNode()
chan.add(n1, n2)
fork = chan.fork()
fork.add(n3)
self.assertTrue(isinstance(fork.message_store, msgstore.NullMessageStore))
whe = chan.when(True, message_store_factory=store_factory)
whe.add(n4)
self.assertTrue(isinstance(whe.message_store, msgstore.MemoryMessageStore))
def test_chan_process_in_test_mode(self):
""" Whether BaseChannel handling with test mode is working """
chan = BaseChannel(name="test_channel_test_1", loop=self.loop)
n = TestNode(name="testme")
chan.add(n)
msg = generate_msg(message_content="X")
# Launch channel processing
self.start_channels()
chan._reset_test()
chan.handle_and_wait(msg)
self.assertEqual(n.processed, 1, "Channel in test mode not working")
self.assertTrue(hasattr(n, '_handle'), "Channel in test mode not working")
self.assertEqual(chan.get_node("testme").last_input().payload, "X", "Last input broken")
def test_loop_slow(self):
""" main loop logs slow tasks """
import pypeman.debug
pypeman.debug.enable_slow_log_stats()
stats = pypeman.debug.stats
self.setup_evt_loop()
# logger = self.logger # get default test logger
# handler = self.loghandler # get test log handler
tst_logger = logging.getLogger('tests.debug.main_loop.slow')
print("HANDLERS", tst_logger.handlers)
loop = self.loop
chan = BaseChannel(name="test_loop_slow", loop=loop)
n1 = SimpleTestNode(delay=0.01, async_delay=0, logger=tst_logger, loop=loop)
n2 = SimpleTestNode(delay=0.12, async_delay=0, logger=tst_logger, loop=loop)
n3 = SimpleTestNode(delay=0.11, async_delay=0, logger=tst_logger, loop=loop)
chan.add(n1)
chan.add(n2)
chan.add(n3)
msg = generate_msg()
# Launch channel processing
t0 = time.time()
print("start channel %.4f" % (time.time() - t0))
self.loop.run_until_complete(chan.start())
print("channel started %.4f" % (time.time() - t0))
self.loop.run_until_complete(chan.handle(msg))
print("msg handled %.4f" % (time.time() - t0))
# handler.show_entries()
self.assertEqual(len(stats), 2, "should have 2 slow tasks, not %d" % len(stats))
self.loop.close()
def test_chan_node_mocking_in_test_mode(self):
""" Whether node mocking input and output is working """
chan = BaseChannel(name="test_channel_test_2", loop=self.loop)
n = TestNode(name="testme")
chan.add(n)
msg_x = generate_msg(message_content="X")
msg_a = generate_msg(message_content="A")
msg_b = generate_msg(message_content="B")
msg_c = generate_msg(message_content="C")
msg_d = generate_msg(message_content="D")
def concat_e(msg):
msg.payload += "E"
return msg
def concat_f(msg):
msg.payload += "F"
return msg
# Launch channel processing
self.start_channels()
# Mock input
chan._reset_test()
chan.get_node("testme").mock(input=msg_a)
ret = chan.handle_and_wait(msg_x)
self.assertEqual(n.processed, 1, "Channel in test mode not working")
self.assertEqual(ret.payload, "A", "Mocking input broken")
self.assertEqual(chan.get_node("testme").last_input().payload, "X", "Last input broken")
# Mock output
chan._reset_test()
chan.get_node("testme").mock(output=msg_b)
ret = chan.handle_and_wait(msg_x)
self.assertEqual(n.processed, 1, "Channel in test mode not working")
self.assertEqual(ret.payload, "B", "Mocking input broken")
self.assertEqual(chan.get_node("testme").last_input().payload, "X", "Last input broken")
# Mock both
chan._reset_test()
chan.get_node("testme").mock(input=msg_c, output=msg_d)
ret = chan.handle_and_wait(msg_x)
self.assertEqual(n.processed, 1, "Channel in test mode not working")
self.assertEqual(ret.payload, "D", "Mocking both input and output broken")
# Mock both functions
chan._reset_test()
chan.get_node("testme").mock(input=concat_e, output=concat_f)
ret = chan.handle_and_wait(msg_x)
self.assertEqual(n.processed, 1, "Channel in test mode not working")
self.assertEqual(ret.payload, "XEF", "Mocking with function broken")
def test_remote_admin_list(self):
""" Channel remote listing working """
store_factory = msgstore.MemoryMessageStoreFactory()
chan = BaseChannel(name="test_remote050", loop=self.loop, message_store_factory=store_factory)
n = TestNode()
n2 = TestNode(name="sub")
n3 = TestNode(name="sub1")
n4 = TestNode(name="sub2")
msg = generate_msg(with_context=True)
msg2 = generate_msg(timestamp=(1982, 11, 27, 12, 35))
msg3 = generate_msg(timestamp=(1982, 11, 28, 12, 35))
msg4 = generate_msg(timestamp=(1982, 11, 28, 14, 35))
idref_msg3 = msg3.uuid
chan.add(n)
sub = chan.fork(name="subchannel")
sub.append(n2, n3, n4)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.loop.run_until_complete(chan.handle(msg2))
self.loop.run_until_complete(chan.handle(msg3))
self.loop.run_until_complete(chan.handle(msg4))
server = RemoteAdminServer(loop=self.loop)
self.loop.run_until_complete(server.start())
client = RemoteAdminClient(loop=self.loop)
client.init()
# List channels
chans = client.channels()
print (chans)
self.assertEqual(chans[0]['name'], 'test_remote050' , "Channel listing not working")
self.assertEqual(chans[0]['subchannels'][0]['name'], 'test_remote050.subchannel' , "Subchannel listing not working")
# Stop channel
result = client.stop('test_remote050')
self.assertEqual(chan.status, BaseChannel.STOPPED, "Stopping channel doesn't work")
# Start channel
result = client.start('test_remote050')
self.assertEqual(chan.status, BaseChannel.WAITING, "Starting channel doesn't work")
# Search message
msg_list = client.list_msg(channel='test_remote050', start=2, count=5, order_by='-timestamp')
print(msg_list)
self.assertEqual(msg_list['total'], 4, 'List channel messages broken')
self.assertEqual(msg_list['messages'][0]['id'], idref_msg3, 'List channel messages broken')
# Replay message
result = client.replay_msg('test_remote050', [idref_msg3])
msg_list = client.list_msg(channel='test_remote050', start=0, count=5, order_by='-timestamp')
self.assertEqual(msg_list['total'], 5, 'List channel messages broken')
self.assertEqual(msg_list['messages'][0]['id'], result[0].uuid, 'Replay messages broken')
# Push message
result = client.push_msg(channel='test_remote050', text="Yaaay")
msg_list = client.list_msg(channel='test_remote050', start=0, count=5, order_by='-timestamp')
self.assertEqual(msg_list['total'], 6, 'Push message broken')
self.assertEqual(msg_list['messages'][0]['id'], result.uuid, 'Push message broken')
def test_memory_message_store(self):
""" We can store a message in FileMessageStore """
store_factory = msgstore.MemoryMessageStoreFactory()
chan = BaseChannel(name="test_channel10", loop=self.loop, message_store_factory=store_factory)
n = TestNode()
n_error = TestConditionalErrorNode()
msg = generate_msg(with_context=True)
msg2 = generate_msg(timestamp=(1982, 11, 27, 12, 35))
msg3 = generate_msg(timestamp=(1982, 11, 28, 12, 35))
msg4 = generate_msg(timestamp=(1982, 11, 28, 14, 35))
# This message should be in error
msg5 = generate_msg(timestamp=(1982, 11, 12, 14, 35))
chan.add(n)
chan.add(n_error)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.loop.run_until_complete(chan.handle(msg2))
self.loop.run_until_complete(chan.handle(msg3))
self.loop.run_until_complete(chan.handle(msg4))
with self.assertRaises(TestException) as cm:
# This message should be in error state
self.loop.run_until_complete(chan.handle(msg5))
self.assertEqual(self.loop.run_until_complete(chan.message_store.total()), 5, "Should be a total of 5 messages in store!")
msg_stored1 = list(self.loop.run_until_complete(chan.message_store.search(0,2)))
self.assertEqual(len(msg_stored1), 2, "Should be 2 results from search!")
msg_stored2 = list(self.loop.run_until_complete(chan.message_store.search(2,5)))
self.assertEqual(len(msg_stored2), 3, "Should be 3 results from search!")
msg_stored = list(self.loop.run_until_complete(chan.message_store.search()))
# All message stored ?
self.assertEqual(len(msg_stored), 5, "Should be 5 messages in store!")
for msg in msg_stored:
print(msg)
for msg in msg_stored1 + msg_stored2:
print(msg)
ids1 = [msg['id'] for msg in msg_stored1 + msg_stored2]
ids2 = [msg['id'] for msg in msg_stored]
self.assertEqual(ids1, ids2, "Should be 5 messages in store!")
# Test processed message
dict_msg = self.loop.run_until_complete(chan.message_store.get('%s' % msg3.uuid))
self.assertEqual(dict_msg['state'], 'processed', "Message %s should be in processed state!" % msg3)
# Test failed message
dict_msg = self.loop.run_until_complete(chan.message_store.get('%s' % msg5.uuid))
self.assertEqual(dict_msg['state'], 'error', "Message %s should be in error state!" % msg5)