def test_replay_from_memory_message_store(self):
""" We can store a message in FileMessageStore """
store_factory = msgstore.MemoryMessageStoreFactory()
chan = BaseChannel(name="test_channel10.5", loop=self.loop, message_store_factory=store_factory)
n = TestNode()
msg = generate_msg(with_context=True)
msg2 = generate_msg(timestamp=(1982, 11, 27, 12, 35))
chan.add(n)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.loop.run_until_complete(chan.handle(msg2))
msg_stored = list(self.loop.run_until_complete(chan.message_store.search()))
for msg in msg_stored:
print(msg)
self.assertEqual(len(msg_stored), 2, "Should be 2 messages in store!")
self.loop.run_until_complete(chan.replay(msg_stored[0]['id']))
msg_stored = list(self.loop.run_until_complete(chan.message_store.search()))
self.assertEqual(len(msg_stored), 3, "Should be 3 messages in store!")
def test_chan_node_mocking_in_test_mode(self):
""" Whether node mocking input and output is working """
chan = BaseChannel(name="test_channel_test_2", loop=self.loop)
n = TestNode(name="testme")
chan.add(n)
msg_x = generate_msg(message_content="X")
msg_a = generate_msg(message_content="A")
msg_b = generate_msg(message_content="B")
msg_c = generate_msg(message_content="C")
msg_d = generate_msg(message_content="D")
def concat_e(msg):
msg.payload += "E"
return msg
def concat_f(msg):
msg.payload += "F"
return msg
# Launch channel processing
self.start_channels()
# Mock input
chan._reset_test()
chan.get_node("testme").mock(input=msg_a)
ret = chan.handle_and_wait(msg_x)
self.assertEqual(n.processed, 1, "Channel in test mode not working")
self.assertEqual(ret.payload, "A", "Mocking input broken")
self.assertEqual(chan.get_node("testme").last_input().payload, "X", "Last input broken")
# Mock output
chan._reset_test()
chan.get_node("testme").mock(output=msg_b)
ret = chan.handle_and_wait(msg_x)
self.assertEqual(n.processed, 1, "Channel in test mode not working")
self.assertEqual(ret.payload, "B", "Mocking input broken")
self.assertEqual(chan.get_node("testme").last_input().payload, "X", "Last input broken")
# Mock both
chan._reset_test()
chan.get_node("testme").mock(input=msg_c, output=msg_d)
ret = chan.handle_and_wait(msg_x)
self.assertEqual(n.processed, 1, "Channel in test mode not working")
self.assertEqual(ret.payload, "D", "Mocking both input and output broken")
# Mock both functions
chan._reset_test()
chan.get_node("testme").mock(input=concat_e, output=concat_f)
ret = chan.handle_and_wait(msg_x)
self.assertEqual(n.processed, 1, "Channel in test mode not working")
self.assertEqual(ret.payload, "XEF", "Mocking with function broken")
def test_file_reader_node(self):
"""if FileReader are functionnal"""
reader = nodes.FileReader(filepath='/filepath', filename='badname')
channel = FakeChannel(self.loop)
reader.channel = channel
msg1 = generate_msg()
with mock.patch("builtins.open", mock.mock_open(read_data="data")) as mock_file:
result = self.loop.run_until_complete(reader.handle(msg1))
mock_file.assert_called_once_with('/filepath', 'r')
self.assertEqual(result.payload, "data", "FileReader not working")
reader2 = nodes.FileReader()
reader2.channel = channel
msg2 = generate_msg()
msg2.meta['filepath'] = '/filepath2'
msg2.meta['filename'] = '/badpath'
with mock.patch("builtins.open", mock.mock_open(read_data="data2")) as mock_file:
result = self.loop.run_until_complete(reader2.handle(msg2))
mock_file.assert_called_once_with('/filepath2', 'r')
self.assertEqual(result.payload, "data2", "FileReader not working with meta")
reader3 = nodes.FileReader(filepath=tstfct, filename='badname')
reader3.channel = channel
msg3 = generate_msg()
msg3.meta['filepath'] = '/badpath'
msg3.meta['filename'] = 'badname2'
with mock.patch("builtins.open", mock.mock_open(read_data="data")) as mock_file:
result = self.loop.run_until_complete(reader3.handle(msg3))
mock_file.assert_called_once_with('/fctpath', 'r')
reader4 = nodes.FileReader(filename=tstfct2)
reader4.channel = channel
msg4 = generate_msg()
msg4.meta['filepath'] = '/filepath3/badname'
msg4.meta['filename'] = 'badname'
with mock.patch("builtins.open", mock.mock_open(read_data="data")) as mock_file:
result = self.loop.run_until_complete(reader4.handle(msg4))
mock_file.assert_called_once_with('/filepath3/fctname', 'r')
def test_sub_channel_with_exception(self):
""" Whether Sub Channel exception handling is working """
chan = BaseChannel(name="test_channel4", loop=self.loop)
n1 = TestNode(name="main")
n2 = TestNode(name="sub")
n3 = ExceptNode(name="sub2")
msg = generate_msg()
chan.add(n1)
sub = chan.fork(name="Hello")
sub.add(n2, n3)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.assertEqual(n1.processed, 1, "Sub Channel not working")
with self.assertRaises(TestException) as cm:
self.clean_loop()
self.assertEqual(n2.processed, 1, "Sub Channel not working")
def test_channel_with_generator(self):
""" Whether BaseChannel with generator is working """
chan = BaseChannel(name="test_channel7.3", loop=self.loop)
chan2 = BaseChannel(name="test_channel7.31", loop=self.loop)
msg = generate_msg()
msg2 = msg.copy()
class TestIter(nodes.BaseNode):
def process(self, msg):
def iter():
for i in range(3):
yield msg
return iter()
final_node = nodes.Log()
mid_node = nodes.Log()
chan.add(TestIter(name="testiterr"), nodes.Log(), TestIter(name="testiterr"), final_node)
chan2.add(TestIter(name="testiterr"), mid_node, nodes.Drop())
# Launch channel processing
self.start_channels()
result = self.loop.run_until_complete(chan.handle(msg))
self.assertEqual(result.payload, msg.payload, "Generator node not working")
self.assertEqual(final_node.processed, 9, "Generator node not working")
result = self.loop.run_until_complete(chan2.handle(msg2))
self.assertEqual(mid_node.processed, 3, "Generator node not working with drop_node")
def test_node_persistence(self):
""" Whether BaseNode() data memory persistence works"""
persistence._backend = None # noqa
# Mock settings
with mock.patch('pypeman.persistence.settings',
conf.Settings(module_name='pypeman.tests.settings.test_settings_persistence')):
result = []
class TestPers(nodes.BaseNode):
""" Test node for persistence """
async def process(self, msg):
await self.save_data('test', 'value')
result.append(await self.restore_data('test'))
result.append(await self.restore_data('titi', default='Yo'))
try:
result.append(await self.restore_data('titi'))
except KeyError:
result.append('yay')
return msg
n = TestPers()
n.channel = FakeChannel(self.loop)
m = generate_msg(message_content='test')
self.loop.run_until_complete(n.handle(m))
self.assertEqual(result[0], 'value', "Can't persist data for node")
self.assertEqual(result[1], 'Yo', "Default value not working")
self.assertEqual(result[2], 'yay', "Exception on missing key not working")
def test_cond_channel(self):
""" Whether Conditionnal channel is working """
chan = BaseChannel(name="test_channel5", loop=self.loop)
n1 = TestNode(name="main")
n2 = TestNode(name="end_main")
not_processed = TestNode(name="cond_notproc")
processed = TestNode(name="cond_proc")
msg = generate_msg()
chan.add(n1)
# Nodes in this channel should not be processed
cond1 = chan.when(lambda x: False, name="Toto")
# Nodes in this channel should be processed
cond2 = chan.when(True, name="condchannel")
chan.add(n2)
cond1.add(not_processed)
cond2.add(processed)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.assertFalse(not_processed.processed, "Cond Channel when condition == False not working")
self.assertTrue(processed.processed, "Cond Channel when condition == True not working")
self.assertFalse(n2.processed, "Cond Channel don't became the main path")
self.assertEqual(cond2.name, "test_channel5.condchannel", "Condchannel name is incorrect")
def test_sub_channel(self):
""" Whether Sub Channel is working """
chan = BaseChannel(name="test_channel3", loop=self.loop)
n1 = TestNode(name="main")
n2 = TestNode(name="sub")
n3 = TestNode(name="sub1")
n4 = TestNode(name="sub2")
msg = generate_msg()
same_chan = chan.append(n1)
self.assertEqual(chan, same_chan, "Append don't return channel.")
sub = chan.fork(name="subchannel")
sub.append(n2, n3, n4)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.assertTrue(n2.processed, "Sub Channel not working")
self.assertTrue(n3.processed, "Sub Channel not working")
self.assertTrue(n4.processed, "Sub Channel not working")
self.assertEqual(sub.name, "test_channel3.subchannel", "Subchannel name is incorrect")
def test_xml_nodes(self):
""" if XML nodes are functional """
try:
import xmltodict # noqa F401
except ImportError:
raise unittest.SkipTest("Missing dependency xmltodict.")
n1 = nodes.XMLToPython()
n2 = nodes.PythonToXML()
channel = FakeChannel(self.loop)
n1.channel = channel
n2.channel = channel
m = generate_msg()
m.payload = '<?xml version="1.0" encoding="utf-8"?>\n<test>hello</test>'
base = str(m.payload)
ret = self.loop.run_until_complete(n1.handle(m))
ext_new = self.loop.run_until_complete(n2.handle(ret))
# Check return
self.assertTrue(isinstance(ret, message.Message))
self.assertEqual(base, ext_new.payload, "XML nodes not working !")
def test_loop_slow2(self):
""" slow tasks delay can be configured """
import pypeman.debug
pypeman.debug.enable_slow_log_stats()
self.setup_evt_loop(slow_cb_duration=0.05)
stats = pypeman.debug.stats
# logger = self.logger # get default test logger
# handler = self.loghandler # get test log handler
tst_logger = logging.getLogger('tests.debug.main_loop.slow')
print(tst_logger.handlers)
loop = self.loop
chan = BaseChannel(name="test_loop_slow2", loop=loop)
n1 = SimpleTestNode(delay=0.03, async_delay=0, logger=tst_logger, loop=loop)
n2 = SimpleTestNode(delay=0.06, logger=tst_logger, loop=loop)
chan.add(n1)
chan.add(n2)
msg = generate_msg()
# Launch channel processing
self.loop.run_until_complete(chan.start())
self.loop.run_until_complete(chan.handle(msg))
# handler.show_entries()
self.assertEqual(len(stats), 1, "should have 1 slow tasks, not %d" % len(stats))
pypeman.debug.show_slow_log_stats()
self.loop.close()
def test_channel_events(self):
""" Whether BaseChannel handling return a good result """
chan = BaseChannel(name="test_channel7.5", loop=self.loop)
msg = generate_msg()
chan.add(nodes.JsonToPython(), nodes.PythonToJson())
state_sequence = [chan.status]
@events.channel_change_state.receiver
def handle_change_state(channel=None, old_state=None, new_state=None):
print(channel.name, old_state, new_state)
state_sequence.append(new_state)
@events.channel_change_state.receiver
async def handle_change_state_async(channel=None, old_state=None, new_state=None):
print(channel.name, old_state, new_state)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.loop.run_until_complete(chan.stop())
print(state_sequence)
valid_sequence = [BaseChannel.STOPPED, BaseChannel.STARTING, BaseChannel.WAITING,
BaseChannel.PROCESSING, BaseChannel.WAITING, BaseChannel.STOPPING, BaseChannel.STOPPED]
self.assertEqual(state_sequence, valid_sequence, "Sequence state is not valid")
def test_case_channel(self):
""" Whether Conditionnal channel is working """
chan = BaseChannel(name="test_channel6", loop=self.loop)
n1 = TestNode(name="main")
n2 = TestNode(name="end_main")
not_processed = TestNode(name="cond_notproc")
processed = TestNode(name="cond_proc")
not_processed2 = TestNode(name="cond_proc2")
msg = generate_msg()
chan.add(n1)
cond1, cond2, cond3 = chan.case(lambda x: False, True, True, names=['first', 'second', 'third'])
chan.add(n2)
cond1.add(not_processed)
cond2.add(processed)
cond3.add(not_processed2)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.assertFalse(not_processed.processed, "Case Channel when condition == False not working")
self.assertFalse(not_processed2.processed, "Case Channel when condition == False not working")
self.assertTrue(processed.processed, "Case Channel when condition == True not working")
self.assertTrue(n2.processed, "Cond Channel don't became the main path")
self.assertEqual(cond1.name, "test_channel6.first", "Casechannel name is incorrect")
self.assertEqual(cond2.name, "test_channel6.second", "Casechannel name is incorrect")
self.assertEqual(cond3.name, "test_channel6.third", "Casechannel name is incorrect")
def test_no_node_base_channel(self):
""" Whether BaseChannel handling is working even if there is no node """
chan = BaseChannel(name="test_channel2", loop=self.loop)
msg = generate_msg()
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
def test_ftp_nodes(self):
""" Whether FTP nodes are functional """
channel = FakeChannel(self.loop)
ftp_config = dict(host="fake", credentials=("fake", "fake"))
fake_ftp = mock.MagicMock()
fake_ftp.download_file = mock.Mock(return_value=b"new_content")
fake_ftp_helper = mock.Mock(return_value=fake_ftp)
with mock.patch('pypeman.contrib.ftp.FTPHelper', new=fake_ftp_helper):
reader = nodes.FTPFileReader(filepath="test_read", **ftp_config)
delete = nodes.FTPFileDeleter(filepath="test_delete", **ftp_config)
writer = nodes.FTPFileWriter(filepath="test_write", **ftp_config)
reader.channel = channel
delete.channel = channel
writer.channel = channel
m1 = generate_msg(message_content="to_be_replaced")
m1_delete = generate_msg(message_content="to_be_replaced")
m2 = generate_msg(message_content="message_content")
# Test reader
result = self.loop.run_until_complete(reader.handle(m1))
fake_ftp.download_file.assert_called_once_with('test_read')
self.assertEqual(result.payload, b"new_content", "FTP reader not working")
# Test reader with delete after
result = self.loop.run_until_complete(delete.handle(m1_delete))
fake_ftp.delete.assert_called_once_with('test_delete')
# test writer
result = self.loop.run_until_complete(writer.handle(m2))
fake_ftp.upload_file.assert_called_once_with('test_write.part', 'message_content')
fake_ftp.rename.assert_called_once_with('test_write.part', 'test_write')
def test_log_node(self):
""" whether Log() node functional """
n = nodes.Log()
n.channel = FakeChannel(self.loop)
m = generate_msg()
ret = self.loop.run_until_complete(n.handle(m))
self.assertTrue(isinstance(ret, message.Message))
def test_channel_exception(self):
""" Whether BaseChannel handling return an exception in case of error in main branch """
chan = BaseChannel(name="test_channel8", loop=self.loop)
msg = generate_msg()
chan.add(nodes.JsonToPython(), nodes.PythonToJson(), ExceptNode())
# Launch channel processing
self.start_channels()
with self.assertRaises(TestException) as cm:
self.loop.run_until_complete(chan.process(msg))
def test_json_to_python_node(self):
""" if JsonToPython() node functional """
n = nodes.JsonToPython()
n.channel = FakeChannel(self.loop)
m = generate_msg(message_content='{"test":1}')
ret = self.loop.run_until_complete(n.handle(m))
# Check return
self.assertTrue(isinstance(ret, message.Message))
self.assertEqual(ret.payload, {"test": 1}, "JsonToPython node not working !")
def test_channel_result(self):
""" Whether BaseChannel handling return a good result """
chan = BaseChannel(name="test_channel7", loop=self.loop)
msg = generate_msg()
chan.add(nodes.JsonToPython(), nodes.PythonToJson())
# Launch channel processing
self.start_channels()
result = self.loop.run_until_complete(chan.handle(msg))
self.assertEqual(result.payload, msg.payload, "Channel handle not working")
def test_sleep_node(self):
""" if Sleep() node functional """
n = nodes.Sleep()
n.channel = FakeChannel(self.loop)
m = generate_msg(message_content='test')
ret = self.loop.run_until_complete(n.handle(m))
# Check return
self.assertTrue(isinstance(ret, message.Message))
self.assertEqual(ret.payload, 'test', "Sleep node not working !")
def test_thread_node(self):
""" if Thread node functional """
# TODO test if another task can be executed in //
n = LongNode()
n.channel = FakeChannel(self.loop)
m = generate_msg()
ret = self.loop.run_until_complete(n.handle(m))
# Check return
self.assertTrue(isinstance(ret, message.Message))
def test_drop_node(self):
""" Whether Drop() node is working """
msg_to_show = "It's only dropped"
n = nodes.Drop(message=msg_to_show)
n.channel = FakeChannel(self.loop)
m = generate_msg(message_content='test')
with self.assertRaises(nodes.Dropped) as cm:
self.loop.run_until_complete(n.handle(m))
self.assertEqual(str(cm.exception), msg_to_show, "Drop node message not working !")
def test_null_message_store(self):
""" We can store a message in NullMessageStore """
chan = BaseChannel(name="test_channel9", loop=self.loop, message_store_factory=msgstore.FakeMessageStoreFactory())
n = TestNode()
msg = generate_msg(with_context=True)
chan.add(n)
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.assertTrue(n.processed, "Channel handle not working")
def test_file_writer_node(self):
"""Whether FileWriter is functionnal"""
writer = nodes.FileWriter(filepath='/filepath', safe_file=False)
channel = FakeChannel(self.loop)
writer.channel = channel
msg1 = generate_msg(message_content="message_content")
with mock.patch("builtins.open", mock.mock_open()) as mock_file:
self.loop.run_until_complete(writer.handle(msg1))
mock_file.assert_called_once_with('/filepath', 'w')
handle = mock_file()
handle.write.assert_called_once_with('message_content')
writer2 = nodes.FileWriter(safe_file=False)
writer.channel = channel
msg2 = generate_msg(message_content="message_content2")
msg2.meta['filepath'] = '/filepath2'
with mock.patch("builtins.open", mock.mock_open()) as mock_file:
self.loop.run_until_complete(writer2.handle(msg2))
mock_file.assert_called_once_with('/filepath2', 'w')
handle = mock_file()
handle.write.assert_called_once_with('message_content2')
def test_base_node(self):
""" if BaseNode() node functional """
n = nodes.BaseNode()
n.channel = FakeChannel(self.loop)
m = generate_msg(message_content='test')
ret = self.loop.run_until_complete(n.handle(m))
# Check return
self.assertTrue(isinstance(ret, message.Message))
self.assertEqual(ret.payload, 'test', "Base node not working !")
self.assertEqual(n.processed, 1, "Processed msg count broken")
def test_channel_stopped_dont_process_message(self):
""" Whether BaseChannel handling return a good result """
chan = BaseChannel(name="test_channel7.7", loop=self.loop)
msg = generate_msg()
chan.add(nodes.JsonToPython(), nodes.PythonToJson())
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.loop.run_until_complete(chan.stop())
with self.assertRaises(channels.ChannelStopped) as cm:
self.loop.run_until_complete(chan.handle(msg))
def test_base_channel(self):
""" Whether BaseChannel handling is working """
chan = BaseChannel(name="test_channel1", loop=self.loop)
n = TestNode()
msg = generate_msg()
same_chan = chan.add(n)
self.assertEqual(same_chan, chan, "Add doesn't return channel")
# Launch channel processing
self.start_channels()
self.loop.run_until_complete(chan.handle(msg))
self.assertTrue(n.processed, "Channel handle not working")
def test_base_logging(self):
""" whether BaseNode() node logging works"""
n = nodes.BaseNode(log_output=True)
n.channel = FakeChannel(self.loop)
m = generate_msg(message_content='test')
ret = self.loop.run_until_complete(n.handle(m))
# Check return
self.assertTrue(isinstance(ret, message.Message))
self.assertEqual(ret.payload, 'test', "Base node not working !")
self.assertEqual(n.processed, 1, "Processed msg count broken")
n.channel.logger.log.assert_any_call(10, 'Payload: %r', 'test')
n.channel.logger.log.assert_called_with(10, 'Meta: %r', {'question': 'unknown'})
def test_email_node(self):
""" if Email() node is functional """
# Set this three vars to enable the skipped test.
# Use mailtrap free account to test it.
recipients = os.environ['PYPEMAN_TEST_RECIPIENT_EMAIL']
smtp_user = os.environ['PYPEMAN_TEST_SMTP_USER']
smtp_password = os.environ['PYPEMAN_TEST_SMTP_PASSWORD']
n = nodes.Email(host="mailtrap.io", port=2525, start_tls=False, ssl=False, user=smtp_user, password=smtp_password,
subject="Sent from email node of Pypeman", sender=recipients, recipients=recipients)
n.channel = FakeChannel(self.loop)
m = generate_msg()
m.payload = "Message content is full of silence !"
ret = self.loop.run_until_complete(n.handle(m))
self.assertTrue(isinstance(ret, message.Message))
def test_chan_process_in_test_mode(self):
""" Whether BaseChannel handling with test mode is working """
chan = BaseChannel(name="test_channel_test_1", loop=self.loop)
n = TestNode(name="testme")
chan.add(n)
msg = generate_msg(message_content="X")
# Launch channel processing
self.start_channels()
chan._reset_test()
chan.handle_and_wait(msg)
self.assertEqual(n.processed, 1, "Channel in test mode not working")
self.assertTrue(hasattr(n, '_handle'), "Channel in test mode not working")
self.assertEqual(chan.get_node("testme").last_input().payload, "X", "Last input broken")
def test_b64_nodes(self):
""" if B64 nodes are functional """
n1 = nodes.B64Encode()
n2 = nodes.B64Decode()
channel = FakeChannel(self.loop)
n1.channel = channel
n2.channel = channel
m = generate_msg()
m.payload = b'hello'
base = bytes(m.payload)
ret = self.loop.run_until_complete(n1.handle(m))
ext_new = self.loop.run_until_complete(n2.handle(ret))
# Check return
self.assertTrue(isinstance(ret, message.Message))
self.assertEqual(base, ext_new.payload, "B64 nodes not working !")
