def test_replay_from_memory_message_store(self): """ We can store a message in FileMessageStore """ store_factory = msgstore.MemoryMessageStoreFactory() chan = BaseChannel(name="test_channel10.5", loop=self.loop, message_store_factory=store_factory) n = TestNode() msg = generate_msg(with_context=True) msg2 = generate_msg(timestamp=(1982, 11, 27, 12, 35)) chan.add(n) # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg)) self.loop.run_until_complete(chan.handle(msg2)) msg_stored = list(self.loop.run_until_complete(chan.message_store.search())) for msg in msg_stored: print(msg) self.assertEqual(len(msg_stored), 2, "Should be 2 messages in store!") self.loop.run_until_complete(chan.replay(msg_stored[0]['id'])) msg_stored = list(self.loop.run_until_complete(chan.message_store.search())) self.assertEqual(len(msg_stored), 3, "Should be 3 messages in store!")
def test_channel_with_generator(self): """ Whether BaseChannel with generator is working """ chan = BaseChannel(name="test_channel7.3", loop=self.loop) chan2 = BaseChannel(name="test_channel7.31", loop=self.loop) msg = generate_msg() msg2 = msg.copy() class TestIter(nodes.BaseNode): def process(self, msg): def iter(): for i in range(3): yield msg return iter() final_node = nodes.Log() mid_node = nodes.Log() chan.add(TestIter(name="testiterr"), nodes.Log(), TestIter(name="testiterr"), final_node) chan2.add(TestIter(name="testiterr"), mid_node, nodes.Drop()) # Launch channel processing self.start_channels() result = self.loop.run_until_complete(chan.handle(msg)) self.assertEqual(result.payload, msg.payload, "Generator node not working") self.assertEqual(final_node.processed, 9, "Generator node not working") result = self.loop.run_until_complete(chan2.handle(msg2)) self.assertEqual(mid_node.processed, 3, "Generator node not working with drop_node")
def test_file_message_store(self): """ We can store a message in FileMessageStore """ tempdir = tempfile.mkdtemp() store_factory = msgstore.FileMessageStoreFactory(path=tempdir) chan = BaseChannel(name="test_channel11", loop=self.loop, message_store_factory=store_factory) n = TestNode() n_error = TestConditionalErrorNode() msg = generate_msg() msg2 = generate_msg(timestamp=(1982, 11, 27, 12, 35)) msg3 = generate_msg(timestamp=(1982, 11, 28, 12, 35)) msg4 = generate_msg(timestamp=(1982, 11, 28, 14, 35)) # This message should be in error msg5 = generate_msg(timestamp=(1982, 11, 12, 14, 35)) chan.add(n) chan.add(n_error) # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg)) self.loop.run_until_complete(chan.handle(msg2)) self.loop.run_until_complete(chan.handle(msg3)) self.loop.run_until_complete(chan.handle(msg4)) with self.assertRaises(TestException) as cm: # This message should be in error state self.loop.run_until_complete(chan.handle(msg5)) msg_stored = list(chan.message_store.search()) for msg in msg_stored: print(msg) # All message stored ? self.assertEqual(len(msg_stored), 5, "Should be 5 messages in store!") # Test processed message dict_msg = chan.message_store.get('1982/11/28/19821128_1235_%s' % msg3.uuid.hex) self.assertEqual(dict_msg['state'], 'processed', "Message %s should be in processed state!" % msg3) # Test failed message dict_msg = chan.message_store.get('1982/11/12/19821112_1435_%s' % msg5.uuid.hex) self.assertEqual(dict_msg['state'], 'error', "Message %s should be in error state!" % msg5) self.assertTrue(os.path.exists("%s/%s/1982/11/28/19821128_1235_%s" % (tempdir, chan.name, msg3.uuid.hex))) # TODO put in tear_down ? shutil.rmtree(tempdir, ignore_errors=True)
def test_channel_stopped_dont_process_message(self): """ Whether BaseChannel handling return a good result """ chan = BaseChannel(name="test_channel7.7", loop=self.loop) msg = generate_msg() chan.add(nodes.JsonToPython(), nodes.PythonToJson()) # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg)) self.loop.run_until_complete(chan.stop()) with self.assertRaises(channels.ChannelStopped) as cm: self.loop.run_until_complete(chan.handle(msg))
def test_memory_message_store(self): """ We can store a message in FileMessageStore """ store_factory = msgstore.MemoryMessageStoreFactory() chan = BaseChannel(name="test_channel10", loop=self.loop, message_store_factory=store_factory) n = TestNode() n_error = TestConditionalErrorNode() msg = generate_msg() msg2 = generate_msg(timestamp=(1982, 11, 27, 12, 35)) msg3 = generate_msg(timestamp=(1982, 11, 28, 12, 35)) msg4 = generate_msg(timestamp=(1982, 11, 28, 14, 35)) # This message should be in error msg5 = generate_msg(timestamp=(1982, 11, 12, 14, 35)) chan.add(n) chan.add(n_error) # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg)) self.loop.run_until_complete(chan.handle(msg2)) self.loop.run_until_complete(chan.handle(msg3)) self.loop.run_until_complete(chan.handle(msg4)) with self.assertRaises(TestException) as cm: # This message should be in error state self.loop.run_until_complete(chan.handle(msg5)) msg_stored = list(chan.message_store.search()) for msg in msg_stored: print(msg) # All message stored ? self.assertEqual(len(msg_stored), 5, "Should be 5 messages in store!") # Test processed message dict_msg = chan.message_store.get('%s' % msg3.uuid.hex) self.assertEqual(dict_msg['state'], 'processed', "Message %s should be in processed state!" % msg3) # Test failed message dict_msg = chan.message_store.get('%s' % msg5.uuid.hex) self.assertEqual(dict_msg['state'], 'error', "Message %s should be in error state!" % msg5)
def test_case_channel(self): """ Whether Conditionnal channel is working """ chan = BaseChannel(name="test_channel6", loop=self.loop) n1 = TestNode(name="main") n2 = TestNode(name="end_main") not_processed = TestNode(name="cond_notproc") processed = TestNode(name="cond_proc") not_processed2 = TestNode(name="cond_proc2") msg = generate_msg() chan.add(n1) cond1, cond2, cond3 = chan.case(lambda x: False, True, True, names=['first', 'second', 'third']) chan.add(n2) cond1.add(not_processed) cond2.add(processed) cond3.add(not_processed2) # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg)) self.assertFalse(not_processed.processed, "Case Channel when condition == False not working") self.assertFalse(not_processed2.processed, "Case Channel when condition == False not working") self.assertTrue(processed.processed, "Case Channel when condition == True not working") self.assertTrue(n2.processed, "Cond Channel don't became the main path") self.assertEqual(cond1.name, "test_channel6.first", "Casechannel name is incorrect") self.assertEqual(cond2.name, "test_channel6.second", "Casechannel name is incorrect") self.assertEqual(cond3.name, "test_channel6.third", "Casechannel name is incorrect")
def test_cond_channel(self): """ Whether Conditionnal channel is working """ chan = BaseChannel(name="test_channel5", loop=self.loop) n1 = TestNode(name="main") n2 = TestNode(name="end_main") not_processed = TestNode(name="cond_notproc") processed = TestNode(name="cond_proc") msg = generate_msg() chan.add(n1) # Nodes in this channel should not be processed cond1 = chan.when(lambda x: False, name="Toto") # Nodes in this channel should be processed cond2 = chan.when(True, name="condchannel") chan.add(n2) cond1.add(not_processed) cond2.add(processed) # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg)) self.assertFalse(not_processed.processed, "Cond Channel when condition == False not working") self.assertTrue(processed.processed, "Cond Channel when condition == True not working") self.assertFalse(n2.processed, "Cond Channel don't became the main path") self.assertEqual(cond2.name, "test_channel5.condchannel", "Condchannel name is incorrect")
def test_sub_channel_with_exception(self): """ Whether Sub Channel exception handling is working """ chan = BaseChannel(name="test_channel4", loop=self.loop) n1 = TestNode(name="main") n2 = TestNode(name="sub") n3 = ExceptNode(name="sub2") msg = generate_msg() chan.add(n1) sub = chan.fork(name="Hello") sub.add(n2, n3) # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg)) self.assertEqual(n1.processed, 1, "Sub Channel not working") with self.assertRaises(TestException) as cm: self.clean_loop() self.assertEqual(n2.processed, 1, "Sub Channel not working")
def test_sub_channel(self): """ Whether Sub Channel is working """ chan = BaseChannel(name="test_channel3", loop=self.loop) n1 = TestNode(name="main") n2 = TestNode(name="sub") n3 = TestNode(name="sub1") n4 = TestNode(name="sub2") msg = generate_msg() same_chan = chan.append(n1) self.assertEqual(chan, same_chan, "Append don't return channel.") sub = chan.fork(name="subchannel") sub.append(n2, n3, n4) # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg)) self.assertTrue(n2.processed, "Sub Channel not working") self.assertTrue(n3.processed, "Sub Channel not working") self.assertTrue(n4.processed, "Sub Channel not working") self.assertEqual(sub.name, "test_channel3.subchannel", "Subchannel name is incorrect")
def test_channel_events(self): """ Whether BaseChannel handling return a good result """ chan = BaseChannel(name="test_channel7.5", loop=self.loop) msg = generate_msg() chan.add(nodes.JsonToPython(), nodes.PythonToJson()) state_sequence = [chan.status] @events.channel_change_state.receiver def handle_change_state(channel=None, old_state=None, new_state=None): print(channel.name, old_state, new_state) state_sequence.append(new_state) @events.channel_change_state.receiver async def handle_change_state_async(channel=None, old_state=None, new_state=None): print(channel.name, old_state, new_state) # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg)) self.loop.run_until_complete(chan.stop()) print(state_sequence) valid_sequence = [BaseChannel.STOPPED, BaseChannel.STARTING, BaseChannel.WAITING, BaseChannel.PROCESSING, BaseChannel.WAITING, BaseChannel.STOPPING, BaseChannel.STOPPED] self.assertEqual(state_sequence, valid_sequence, "Sequence state is not valid")
def test_loop_slow2(self): """ slow tasks delay can be configured """ import pypeman.debug pypeman.debug.enable_slow_log_stats() self.setup_evt_loop(slow_cb_duration=0.05) stats = pypeman.debug.stats # logger = self.logger # get default test logger # handler = self.loghandler # get test log handler tst_logger = logging.getLogger('tests.debug.main_loop.slow') print(tst_logger.handlers) loop = self.loop chan = BaseChannel(name="test_loop_slow2", loop=loop) n1 = SimpleTestNode(delay=0.03, async_delay=0, logger=tst_logger, loop=loop) n2 = SimpleTestNode(delay=0.06, logger=tst_logger, loop=loop) chan.add(n1) chan.add(n2) msg = generate_msg() # Launch channel processing self.loop.run_until_complete(chan.start()) self.loop.run_until_complete(chan.handle(msg)) # handler.show_entries() self.assertEqual(len(stats), 1, "should have 1 slow tasks, not %d" % len(stats)) pypeman.debug.show_slow_log_stats() self.loop.close()
def test_no_node_base_channel(self): """ Whether BaseChannel handling is working even if there is no node """ chan = BaseChannel(name="test_channel2", loop=self.loop) msg = generate_msg() # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg))
def test_channel_result(self): """ Whether BaseChannel handling return a good result """ chan = BaseChannel(name="test_channel7", loop=self.loop) msg = generate_msg() chan.add(nodes.JsonToPython(), nodes.PythonToJson()) # Launch channel processing self.start_channels() result = self.loop.run_until_complete(chan.handle(msg)) self.assertEqual(result.payload, msg.payload, "Channel handle not working")
def test_base_channel(self): """ Whether BaseChannel handling is working """ chan = BaseChannel(name="test_channel1", loop=self.loop) n = TestNode() msg = generate_msg() chan.add(n) # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg)) self.assertTrue(n.processed, "Channel handle not working")
def test_null_message_store(self): """ We can store a message in NullMessageStore """ chan = BaseChannel(name="test_channel9", loop=self.loop, message_store_factory=msgstore.FakeMessageStoreFactory()) n = TestNode() msg = generate_msg(with_context=True) chan.add(n) # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg)) self.assertTrue(n.processed, "Channel handle not working")
def test_loop_slow(self): """ main loop logs slow tasks """ import pypeman.debug pypeman.debug.enable_slow_log_stats() stats = pypeman.debug.stats self.setup_evt_loop() # logger = self.logger # get default test logger # handler = self.loghandler # get test log handler tst_logger = logging.getLogger('tests.debug.main_loop.slow') print("HANDLERS", tst_logger.handlers) loop = self.loop chan = BaseChannel(name="test_loop_slow", loop=loop) n1 = SimpleTestNode(delay=0.01, async_delay=0, logger=tst_logger, loop=loop) n2 = SimpleTestNode(delay=0.12, async_delay=0, logger=tst_logger, loop=loop) n3 = SimpleTestNode(delay=0.11, async_delay=0, logger=tst_logger, loop=loop) chan.add(n1) chan.add(n2) chan.add(n3) msg = generate_msg() # Launch channel processing t0 = time.time() print("start channel %.4f" % (time.time() - t0)) self.loop.run_until_complete(chan.start()) print("channel started %.4f" % (time.time() - t0)) self.loop.run_until_complete(chan.handle(msg)) print("msg handled %.4f" % (time.time() - t0)) # handler.show_entries() self.assertEqual(len(stats), 2, "should have 2 slow tasks, not %d" % len(stats)) self.loop.close()
def test_memory_message_store(self): """ We can store a message in FileMessageStore """ store_factory = msgstore.MemoryMessageStoreFactory() chan = BaseChannel(name="test_channel10", loop=self.loop, message_store_factory=store_factory) n = TestNode() n_error = TestConditionalErrorNode() msg = generate_msg(with_context=True) msg2 = generate_msg(timestamp=(1982, 11, 27, 12, 35)) msg3 = generate_msg(timestamp=(1982, 11, 28, 12, 35)) msg4 = generate_msg(timestamp=(1982, 11, 28, 14, 35)) # This message should be in error msg5 = generate_msg(timestamp=(1982, 11, 12, 14, 35)) chan.add(n) chan.add(n_error) # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg)) self.loop.run_until_complete(chan.handle(msg2)) self.loop.run_until_complete(chan.handle(msg3)) self.loop.run_until_complete(chan.handle(msg4)) with self.assertRaises(TestException) as cm: # This message should be in error state self.loop.run_until_complete(chan.handle(msg5)) self.assertEqual(self.loop.run_until_complete(chan.message_store.total()), 5, "Should be a total of 5 messages in store!") msg_stored1 = list(self.loop.run_until_complete(chan.message_store.search(0,2))) self.assertEqual(len(msg_stored1), 2, "Should be 2 results from search!") msg_stored2 = list(self.loop.run_until_complete(chan.message_store.search(2,5))) self.assertEqual(len(msg_stored2), 3, "Should be 3 results from search!") msg_stored = list(self.loop.run_until_complete(chan.message_store.search())) # All message stored ? self.assertEqual(len(msg_stored), 5, "Should be 5 messages in store!") for msg in msg_stored: print(msg) for msg in msg_stored1 + msg_stored2: print(msg) ids1 = [msg['id'] for msg in msg_stored1 + msg_stored2] ids2 = [msg['id'] for msg in msg_stored] self.assertEqual(ids1, ids2, "Should be 5 messages in store!") # Test processed message dict_msg = self.loop.run_until_complete(chan.message_store.get('%s' % msg3.uuid)) self.assertEqual(dict_msg['state'], 'processed', "Message %s should be in processed state!" % msg3) # Test failed message dict_msg = self.loop.run_until_complete(chan.message_store.get('%s' % msg5.uuid)) self.assertEqual(dict_msg['state'], 'error', "Message %s should be in error state!" % msg5)
def test_remote_admin_list(self): """ Channel remote listing working """ store_factory = msgstore.MemoryMessageStoreFactory() chan = BaseChannel(name="test_remote050", loop=self.loop, message_store_factory=store_factory) n = TestNode() n2 = TestNode(name="sub") n3 = TestNode(name="sub1") n4 = TestNode(name="sub2") msg = generate_msg(with_context=True) msg2 = generate_msg(timestamp=(1982, 11, 27, 12, 35)) msg3 = generate_msg(timestamp=(1982, 11, 28, 12, 35)) msg4 = generate_msg(timestamp=(1982, 11, 28, 14, 35)) idref_msg3 = msg3.uuid chan.add(n) sub = chan.fork(name="subchannel") sub.append(n2, n3, n4) # Launch channel processing self.start_channels() self.loop.run_until_complete(chan.handle(msg)) self.loop.run_until_complete(chan.handle(msg2)) self.loop.run_until_complete(chan.handle(msg3)) self.loop.run_until_complete(chan.handle(msg4)) server = RemoteAdminServer(loop=self.loop) self.loop.run_until_complete(server.start()) client = RemoteAdminClient(loop=self.loop) client.init() # List channels chans = client.channels() print (chans) self.assertEqual(chans[0]['name'], 'test_remote050' , "Channel listing not working") self.assertEqual(chans[0]['subchannels'][0]['name'], 'test_remote050.subchannel' , "Subchannel listing not working") # Stop channel result = client.stop('test_remote050') self.assertEqual(chan.status, BaseChannel.STOPPED, "Stopping channel doesn't work") # Start channel result = client.start('test_remote050') self.assertEqual(chan.status, BaseChannel.WAITING, "Starting channel doesn't work") # Search message msg_list = client.list_msg(channel='test_remote050', start=2, count=5, order_by='-timestamp') print(msg_list) self.assertEqual(msg_list['total'], 4, 'List channel messages broken') self.assertEqual(msg_list['messages'][0]['id'], idref_msg3, 'List channel messages broken') # Replay message result = client.replay_msg('test_remote050', [idref_msg3]) msg_list = client.list_msg(channel='test_remote050', start=0, count=5, order_by='-timestamp') self.assertEqual(msg_list['total'], 5, 'List channel messages broken') self.assertEqual(msg_list['messages'][0]['id'], result[0].uuid, 'Replay messages broken') # Push message result = client.push_msg(channel='test_remote050', text="Yaaay") msg_list = client.list_msg(channel='test_remote050', start=0, count=5, order_by='-timestamp') self.assertEqual(msg_list['total'], 6, 'Push message broken') self.assertEqual(msg_list['messages'][0]['id'], result.uuid, 'Push message broken')