Exemplo n.º 1
0
    def setUp(self):
        self.msg_server = OnDemandPublisher()
        self.msg_server.start()
        self.deliveryDest = functools.partial(self._deliveryDest)
        self.bus = BusAdapter()
        self.delivery_event = threading.Event()

        # Content and topic for outgoing msgs sent
        # via the OnDemandPublisher instance:

        self.reference_msg_content = 'rTmzntNSQLmXokesBpLmAbPYeysftXnuntfdPKrxMVNUuqVFHFzfcrrSaRssdHuMRhPYjXKYrJjwKcyeYycEzQSkJubTabeSFLRS'
        self.reference_named_topic = 'MyTopic'
        self.reference_pattern_topic = r'MyTopic*'
        self.reference_pattern_obj = re.compile(self.reference_pattern_topic)
        self.reference_pattern_matching_topic = 'MyTopicHurray'
        self.reference_context = 'myContext'

        # Msg with a non-pattern, i.e. fixed-name topic:
        self.reference_bus_msg = BusMessage(
            topicName=self.reference_named_topic,
            content=self.reference_msg_content,
            context=self.reference_context)

        # Msg with a topic that will match self.reference_pattern_topic:
        self.reference_bus_wildcard_msg = BusMessage(
            topicName=self.reference_pattern_matching_topic,
            content=self.reference_msg_content,
            context=self.reference_context)
Exemplo n.º 2
0
class MsgFileWriter(object):
    '''
    classdocs
    '''


    def __init__(self, topic_to_follow):
        '''
        Constructor
        '''
        target_file = '/tmp/msgFile.txt'
        self.bus = BusAdapter()
        
        # Make sure to start with a new file
        try:
            os.remove(target_file)
        except:
            pass
        
        self.fd  = open(target_file, 'a') 
        self.bus.subscribeToTopic(topic_to_follow, functools.partial(self.handle_bus_msg))
        
    def handle_bus_msg(self, bus_msg):
        self.fd.write(bus_msg.content + '\n')
        
    def shutdown(self):
        self.bus.close()
Exemplo n.º 3
0
class MsgFileWriter(object):
    '''
    classdocs
    '''
    def __init__(self, topic_to_follow):
        '''
        Constructor
        '''
        target_file = '/tmp/msgFile.txt'
        self.bus = BusAdapter()

        # Make sure to start with a new file
        try:
            os.remove(target_file)
        except:
            pass

        self.fd = open(target_file, 'a')
        self.bus.subscribeToTopic(topic_to_follow,
                                  functools.partial(self.handle_bus_msg))

    def handle_bus_msg(self, bus_msg):
        self.fd.write(bus_msg.content + '\n')

    def shutdown(self):
        self.bus.close()
Exemplo n.º 4
0
    def __init__(self):
        '''
        Constructor
        '''
        self.host = 'localhost'
        self.port = 6379

        self.letterChoice = string.letters
        self.bus = BusAdapter()
Exemplo n.º 5
0
    def __init__(self, busMsg=None):
        super(MessageOutStreamer, self).__init__()

        self.daemon = True
        self.streamBus = BusAdapter()
        self.busMsg = BusMessage() if busMsg is None else busMsg

        self.done = False
        self._paused = False
        self._stream_interval = STREAM_INTERVAL
Exemplo n.º 6
0
class ReceptionTester(threading.Thread):
    '''
    Thread that receives messages, and asserts
    that the received values are what was passed
    into the thread's init method. Keeps listening
    till stop() is called.
    '''
    def __init__(self,
                 msgMd5=None,
                 beSynchronous=False,
                 topic_to_wait_on=RedisPerformanceTester.PUBLISH_TOPIC):
        threading.Thread.__init__(self, name='PerfTestReceptor')
        self.daemon = True

        self.rxed_count = 0
        self.beSynchronous = beSynchronous

        self.testBus = BusAdapter()

        # Subscribe, and ensure that context is delivered
        # with each message:
        self.testBus.subscribeToTopic(topic_to_wait_on,
                                      functools.partial(self.messageReceiver),
                                      context=msgMd5)
        self.interruptEvent = threading.Event()
        self.done = False

    def messageReceiver(self, busMsg, context=None):
        '''
        Method that is called with each received message.
        
        :param busMsg: bus message object
        :type busMsg: BusMessage
        :param context: context Python structure, if subscribeToTopic() was
            called with one.
        :type context: <any>
        '''
        if context is not None:
            inMd5 = hashlib.md5(str(busMsg.content)).hexdigest()
            # Check that the context was delivered:
            if inMd5 != context:
                raise ValueError("md5 in msg should be %s, but was %s" %
                                 (inMd5, context))

        self.rxed_count += 1
        if self.rxed_count % 1000 == 0:
            print(self.rxed_count)

        if self.beSynchronous:
            # Publish a response:
            self.testBus.publish(self.testBus.makeResponseMsg(busMsg),
                                 busMsg.content)

    def stop(self):
        self.interruptEvent.set()

    def run(self):
        self.interruptEvent.wait()
        self.testBus.unsubscribeFromTopic(RedisPerformanceTester.PUBLISH_TOPIC)
        self.testBus.close()
Exemplo n.º 7
0
class ReceptionTester(threading.Thread):
    '''
    Thread that receives messages, and asserts
    that the received values are what was passed
    into the thread's init method. Keeps listening
    till stop() is called.
    '''
    def __init__(self,
                 msgMd5=None,
                 beSynchronous=False,
                 topic_to_wait_on='test'):
        threading.Thread.__init__(self, name='PerfTestReceptor')
        self.setDaemon(True)

        self.beSynchronous = beSynchronous
        self.topic_to_wait_on = topic_to_wait_on

        self.testBus = BusAdapter()

        # Subscribe, and ensure that context is delivered
        # with each message:
        self.testBus.subscribeToTopic(topic_to_wait_on,
                                      functools.partial(self.messageReceiver),
                                      context=msgMd5)
        self.interruptEvent = threading.Event()
        self.done = False

    def messageReceiver(self, busMsg, context=None):
        '''
        Method that is called with each received message.
        
        :param busMsg: bus message object
        :type busMsg: BusMessage
        :param context: context Python structure, if subscribeToTopic() was
            called with one.
        :type context: <any>
        '''

        #         inMd5 = hashlib.md5(str(busMsg.content)).hexdigest()
        #         # Check that the context was delivered:
        #         if inMd5 != context:
        #             raise ValueError("md5 in msg should be %s, but was %s" % (inMd5, context))

        if self.beSynchronous:
            # Publish a response:
            self.testBus.publish(
                self.testBus.makeResponseMsg(busMsg, busMsg.content))

    def stop(self, signum=None, frame=None):
        #**********
        print('Cntr-C called')
        #**********
        self.interruptEvent.set()

    def run(self):
        print("Sync-call test server started; listening on %s; Cnt-C to quit" %
              self.topic_to_wait_on)
        self.interruptEvent.wait()
        self.testBus.unsubscribeFromTopic('test')
        self.testBus.close()
Exemplo n.º 8
0
class ReceptionTester(threading.Thread):
    '''
    Thread that receives messages, and asserts
    that the received values are what was passed
    into the thread's init method. Keeps listening
    till stop() is called.
    '''
    
    def __init__(self, msgMd5=None, beSynchronous=False, topic_to_wait_on=RedisPerformanceTester.PUBLISH_TOPIC):
        threading.Thread.__init__(self, name='PerfTestReceptor')
        self.daemon = True
       
        self.rxed_count = 0
        self.beSynchronous = beSynchronous
        
        self.testBus = BusAdapter()
        
        # Subscribe, and ensure that context is delivered
        # with each message:
        self.testBus.subscribeToTopic(topic_to_wait_on, 
                                      functools.partial(self.messageReceiver), 
                                      context=msgMd5)
        self.interruptEvent = threading.Event()
        self.done = False
                
    def messageReceiver(self, busMsg, context=None):
        '''
        Method that is called with each received message.
        
        :param busMsg: bus message object
        :type busMsg: BusMessage
        :param context: context Python structure, if subscribeToTopic() was
            called with one.
        :type context: <any>
        '''
        if context is not None:
            inMd5 = hashlib.md5(str(busMsg.content)).hexdigest()
            # Check that the context was delivered:
            if inMd5 != context:
                raise ValueError("md5 in msg should be %s, but was %s" % (inMd5, context))

        self.rxed_count += 1
        if self.rxed_count % 1000 == 0:
            print(self.rxed_count)
        
        if self.beSynchronous:
            # Publish a response:
            self.testBus.publish(self.testBus.makeResponseMsg(busMsg), busMsg.content)

    
    def stop(self):
        self.interruptEvent.set()
            
    def run(self):
        self.interruptEvent.wait()
        self.testBus.unsubscribeFromTopic(RedisPerformanceTester.PUBLISH_TOPIC)
        self.testBus.close()
Exemplo n.º 9
0
class ReceptionTester(threading.Thread):
    '''
    Thread that receives messages, and asserts
    that the received values are what was passed
    into the thread's init method. Keeps listening
    till stop() is called.
    '''
    
    def __init__(self, msgMd5=None, beSynchronous=False, topic_to_wait_on='test'):
        threading.Thread.__init__(self, name='PerfTestReceptor')
        self.setDaemon(True)
        
        self.beSynchronous = beSynchronous
        self.topic_to_wait_on = topic_to_wait_on
        
        self.testBus = BusAdapter()
        
        # Subscribe, and ensure that context is delivered
        # with each message:
        self.testBus.subscribeToTopic(topic_to_wait_on, 
                                      functools.partial(self.messageReceiver), 
                                      context=msgMd5)
        self.interruptEvent = threading.Event()
        self.done = False
                
    def messageReceiver(self, busMsg, context=None):
        '''
        Method that is called with each received message.
        
        :param busMsg: bus message object
        :type busMsg: BusMessage
        :param context: context Python structure, if subscribeToTopic() was
            called with one.
        :type context: <any>
        '''
        
#         inMd5 = hashlib.md5(str(busMsg.content)).hexdigest()
#         # Check that the context was delivered:
#         if inMd5 != context:
#             raise ValueError("md5 in msg should be %s, but was %s" % (inMd5, context))
                
        if self.beSynchronous:
            # Publish a response:
            self.testBus.publish(self.testBus.makeResponseMsg(busMsg, busMsg.content))

    
    def stop(self, signum=None, frame=None):
        #**********
        print('Cntr-C called')
        #**********
        self.interruptEvent.set()
            
    def run(self):
        print("Sync-call test server started; listening on %s; Cnt-C to quit" % self.topic_to_wait_on)
        self.interruptEvent.wait()
        self.testBus.unsubscribeFromTopic('test')
        self.testBus.close()
Exemplo n.º 10
0
    def __init__(self):
        '''
        Constructor
        '''
        self.bus = BusAdapter()
        self.bus.subscribeToTopic(SchoolbusWikipedia.TOPIC,
                                  functools.partial(self.get_info_handler))

        # Hang till keyboard_interrupt:
        try:
            self.exit_event = threading.Event().wait()
        except KeyboardInterrupt:
            print('Exiting wikipedia module.')
 def setUp(self):
     self.msg_server = OnDemandPublisher()
     self.msg_server.start()
     self.deliveryDest = functools.partial(self._deliveryDest)
     self.bus = BusAdapter()
     self.delivery_event = threading.Event()
     
     # Content and topic for outgoing msgs sent
     # via the OnDemandPublisher instance:
     
     self.reference_msg_content = 'rTmzntNSQLmXokesBpLmAbPYeysftXnuntfdPKrxMVNUuqVFHFzfcrrSaRssdHuMRhPYjXKYrJjwKcyeYycEzQSkJubTabeSFLRS'
     self.reference_named_topic = 'MyTopic'
     self.reference_pattern_topic = r'MyTopic*'
     self.reference_pattern_obj = re.compile(self.reference_pattern_topic)
     self.reference_pattern_matching_topic = 'MyTopicHurray'
     self.reference_context = 'myContext'
     
     # Msg with a non-pattern, i.e. fixed-name topic:
     self.reference_bus_msg = BusMessage(topicName=self.reference_named_topic, 
                                         content=self.reference_msg_content, 
                                         context=self.reference_context)
     
     # Msg with a topic that will match self.reference_pattern_topic: 
     self.reference_bus_wildcard_msg = BusMessage(topicName=self.reference_pattern_matching_topic, 
                                                  content=self.reference_msg_content,
                                                  context=self.reference_context) 
Exemplo n.º 12
0
    def __init__(self, topic_to_follow):
        '''
        Constructor
        '''
        target_file = '/tmp/msgFile.txt'
        self.bus = BusAdapter()

        # Make sure to start with a new file
        try:
            os.remove(target_file)
        except:
            pass

        self.fd = open(target_file, 'a')
        self.bus.subscribeToTopic(topic_to_follow,
                                  functools.partial(self.handle_bus_msg))
Exemplo n.º 13
0
 def __init__(self):
     '''
     Constructor
     '''
     self.host = 'localhost'
     self.port = 6379
     
     self.letterChoice = string.letters
     self.bus = BusAdapter()
Exemplo n.º 14
0
    def __init__(self, correctValue=None, beSynchronous=False):
        threading.Thread.__init__(self, name='ReceptionTesterThread')
        self.correctValue = correctValue
        self.beSynchronous = beSynchronous

        self.testBus = BusAdapter()

        # Subscribe, and ensure that context is delivered
        # with each message:
        self.testBus.subscribeToTopic('myTopic',
                                      deliveryCallback=functools.partial(
                                          self.messageReceiver),
                                      context={
                                          'foo': 10,
                                          'bar': 'my string'
                                      })
        self.interruptEvent = threading.Event()
        self.done = False
Exemplo n.º 15
0
    def __init__(self,
                 msgMd5=None,
                 beSynchronous=False,
                 topic_to_wait_on='test'):
        threading.Thread.__init__(self, name='PerfTestReceptor')
        self.setDaemon(True)

        self.beSynchronous = beSynchronous
        self.topic_to_wait_on = topic_to_wait_on

        self.testBus = BusAdapter()

        # Subscribe, and ensure that context is delivered
        # with each message:
        self.testBus.subscribeToTopic(topic_to_wait_on,
                                      functools.partial(self.messageReceiver),
                                      context=msgMd5)
        self.interruptEvent = threading.Event()
        self.done = False
 def __init__(self, busMsg=None):
     super(MessageOutStreamer, self).__init__()
     
     self.daemon = True
     self.streamBus = BusAdapter()
     self.busMsg = BusMessage() if busMsg is None else busMsg
 
     self.done = False
     self._paused = False
     self._stream_interval = STREAM_INTERVAL
Exemplo n.º 17
0
    def __init__(self,
                 msgMd5=None,
                 beSynchronous=False,
                 topic_to_wait_on=RedisPerformanceTester.PUBLISH_TOPIC):
        threading.Thread.__init__(self, name='PerfTestReceptor')
        self.daemon = True

        self.rxed_count = 0
        self.beSynchronous = beSynchronous

        self.testBus = BusAdapter()

        # Subscribe, and ensure that context is delivered
        # with each message:
        self.testBus.subscribeToTopic(topic_to_wait_on,
                                      functools.partial(self.messageReceiver),
                                      context=msgMd5)
        self.interruptEvent = threading.Event()
        self.done = False
Exemplo n.º 18
0
class ReceptionTester(threading.Thread):
    '''
    Thread that receives messages, and asserts
    that the received values are what was passed
    into the thread's init method. Keeps listening
    till stop() is called.
    '''
    def __init__(self, correctValue=None, beSynchronous=False):
        threading.Thread.__init__(self, name='ReceptionTesterThread')
        self.correctValue = correctValue
        self.beSynchronous = beSynchronous

        self.testBus = BusAdapter()

        # Subscribe, and ensure that context is delivered
        # with each message:
        self.testBus.subscribeToTopic('myTopic',
                                      deliveryCallback=functools.partial(
                                          self.messageReceiver),
                                      context={
                                          'foo': 10,
                                          'bar': 'my string'
                                      })
        self.interruptEvent = threading.Event()
        self.done = False

    def messageReceiver(self, busMsg, context=None):
        '''
        Method that is called with each received message.
        
        :param busMsg: bus message object
        :type busMsg: BusMessage
        :param context: context Python structure, if subscribeToTopic() was
            called with one.
        :type context: <any>
        '''

        # Check that the context was delivered:
        assert (context['foo'] == 10)
        assert (context['bar'] == 'my string')

        if self.beSynchronous:
            # Publish a response: doubling the int-ified content
            # of the incoming msg:
            response = int(busMsg.content) * 2
            self.testBus.publish(self.testBus.makeResponseMsg(
                busMsg, response))
        else:
            assert (busMsg.content == self.correctValue)

    def stop(self):
        self.interruptEvent.set()

    def run(self):
        self.interruptEvent.wait()
        self.testBus.close()
Exemplo n.º 19
0
 def __init__(self):
     '''
     Constructor
     '''
     self.bus = BusAdapter()
     self.bus.subscribeToTopic(SchoolbusWikipedia.TOPIC, 
                               functools.partial(self.get_info_handler))
     
     # Hang till keyboard_interrupt:
     try:
         self.exit_event = threading.Event().wait()
     except KeyboardInterrupt:
         print('Exiting wikipedia module.')
Exemplo n.º 20
0
 def setUpClass(cls):
     super(LtiBridgeTester, cls).setUpClass()
     cls.bus = BusAdapter()
     if not is_running('lti_schoolbus_bridge'):
         cls.bridge_was_running = False
         # We assume that the lti bridge executable is 
         # in the parent directory:
         currDir = os.path.dirname(__file__)
         path = os.path.join(currDir, '../lti_schoolbus_bridge.py')
         subprocess.Popen(path, shell=True)
         print('Started lti_schoolbus_bridge.py')
     else:
         cls.bridge_was_running = True
Exemplo n.º 21
0
class ReceptionTester(threading.Thread):
    '''
    Thread that receives messages, and asserts
    that the received values are what was passed
    into the thread's init method. Keeps listening
    till stop() is called.
    '''
    
    def __init__(self, correctValue=None, beSynchronous=False):
        threading.Thread.__init__(self, name='ReceptionTesterThread')
        self.correctValue = correctValue
        self.beSynchronous = beSynchronous
        
        self.testBus = BusAdapter()
        
        # Subscribe, and ensure that context is delivered
        # with each message:
        self.testBus.subscribeToTopic('myTopic', 
                                      deliveryCallback=functools.partial(self.messageReceiver), 
                                      context={'foo' : 10, 'bar' : 'my string'})
        self.interruptEvent = threading.Event()
        self.done = False
        
    def messageReceiver(self, busMsg, context=None):
        '''
        Method that is called with each received message.
        
        :param busMsg: bus message object
        :type busMsg: BusMessage
        :param context: context Python structure, if subscribeToTopic() was
            called with one.
        :type context: <any>
        '''
        
        # Check that the context was delivered:
        assert(context['foo'] == 10)
        assert(context['bar'] == 'my string')
        
        if self.beSynchronous:
            # Publish a response: doubling the int-ified content
            # of the incoming msg:
            response = int(busMsg.content) * 2
            self.testBus.publish(self.testBus.makeResponseMsg(busMsg, response))
        else:
            assert(busMsg.content == self.correctValue)

    
    def stop(self):
        self.interruptEvent.set()
            
    def run(self):
        self.interruptEvent.wait()
        self.testBus.close()
Exemplo n.º 22
0
 def __init__(self, correctValue=None, beSynchronous=False):
     threading.Thread.__init__(self, name='ReceptionTesterThread')
     self.correctValue = correctValue
     self.beSynchronous = beSynchronous
     
     self.testBus = BusAdapter()
     
     # Subscribe, and ensure that context is delivered
     # with each message:
     self.testBus.subscribeToTopic('myTopic', 
                                   deliveryCallback=functools.partial(self.messageReceiver), 
                                   context={'foo' : 10, 'bar' : 'my string'})
     self.interruptEvent = threading.Event()
     self.done = False
Exemplo n.º 23
0
 def __init__(self, topic_to_follow):
     '''
     Constructor
     '''
     target_file = '/tmp/msgFile.txt'
     self.bus = BusAdapter()
     
     # Make sure to start with a new file
     try:
         os.remove(target_file)
     except:
         pass
     
     self.fd  = open(target_file, 'a') 
     self.bus.subscribeToTopic(topic_to_follow, functools.partial(self.handle_bus_msg))
Exemplo n.º 24
0
 def initialize(self):
     '''
     This method is call once when the server is started. In 
     contrast, the __init__() method, which we don't override
     in this class is called every time a new request arrives.
     '''
     
     # Create a BusAdapter instance that handles all
     # interactions with the SchoolBus:
     self.busAdapter = BusAdapter()
     
     # Create or read existing JSON file with all
     # subscriptions:
     try:
         self.lti_subscriptions = JsonFileDict(LTISchoolbusBridge.subscriptions_path)
         self.lti_subscriptions.load()
     except (ValueError, IOError):
         # The persistent-subscription file was absent,
         # or contained non-JSON:
         try:
             with open(LTISchoolbusBridge.subscriptions_path, 'r') as fd:
                 subscriptions_raw = fd.readlines()
             if subscriptions_raw is not None and len(subscriptions_raw) > 0:
                 self.logErr('Bad JSON in subscription file %s: %s' % (LTISchoolbusBridge.subscriptions_path,
                                                                       str(subscriptions_raw)))
         except Exception:
             # Can't even read the subscription file:
             self.logErr('Could not read subscription file %s' % LTISchoolbusBridge.subscriptions_path)
         with open(LTISchoolbusBridge.subscriptions_path, 'w') as fd:
             fd.write('{}')
         self.lti_subscriptions = {}
     self.logInfo('Loaded existing subscriptions: %s' %\
                   str(self.lti_subscriptions) if len(self.lti_subscriptions) > 0 else 'No subscriptions on record.')
     
     # Callback for BusAdapter when a message arrives
     # on the bus, destined for an LTI end point:
     self.bus_in_msg_callback = functools.partial(self.bus_to_lti_callback)
     # Bus-inmsg-handler:
     self.bus_in_msg_handler  = functools.partial(self.to_lti_transmitter)
     
     self.published_to_bus_counter = 0
     self.delivered_to_lti_counter = 0
     
     # If there are subscriptions from last time this
     # server ran, then re-subscribe to them:
     for bus_topic in self.lti_subscriptions.keys():
         self.logInfo('Subscribing to bus topic %s' % bus_topic)
         self.busAdapter.subscribeToTopic(bus_topic, self.bus_in_msg_callback)
Exemplo n.º 25
0
 def __init__(self, msgMd5=None, beSynchronous=False, topic_to_wait_on='test'):
     threading.Thread.__init__(self, name='PerfTestReceptor')
     self.setDaemon(True)
     
     self.beSynchronous = beSynchronous
     self.topic_to_wait_on = topic_to_wait_on
     
     self.testBus = BusAdapter()
     
     # Subscribe, and ensure that context is delivered
     # with each message:
     self.testBus.subscribeToTopic(topic_to_wait_on, 
                                   functools.partial(self.messageReceiver), 
                                   context=msgMd5)
     self.interruptEvent = threading.Event()
     self.done = False
Exemplo n.º 26
0
 def __init__(self, msgMd5=None, beSynchronous=False, topic_to_wait_on=RedisPerformanceTester.PUBLISH_TOPIC):
     threading.Thread.__init__(self, name='PerfTestReceptor')
     self.daemon = True
    
     self.rxed_count = 0
     self.beSynchronous = beSynchronous
     
     self.testBus = BusAdapter()
     
     # Subscribe, and ensure that context is delivered
     # with each message:
     self.testBus.subscribeToTopic(topic_to_wait_on, 
                                   functools.partial(self.messageReceiver), 
                                   context=msgMd5)
     self.interruptEvent = threading.Event()
     self.done = False
Exemplo n.º 27
0
class LTISchoolbusBridge(tornado.web.RequestHandler):
    '''
    Operates on two communication systems at once:
    HTTP, and a SchoolBus. Information is received
    on incoming HTTP POST requests that are expected
    to be LTI protocol. Contents of POST requests are
    forwarded by being published to the SchoolBus.
    
    Information flow in the reverse direction requires
    the LTI consumer to supply a delivery URL where it
    is ready to receive POSTs. The POST bodys will have
    this format:
            {
                "time"   : "ISO time string",
                "bus_topic"  : "SchoolBus topic of bus message",
                "payload": "message's 'content' field"
            }    
    
    TODO: check against LTI 1.1 conventions 
           (see https://canvas.instructure.com/doc/api/file.assignment_tools.html).    
    
    The Web service that handles POST requests from 
    an LTI consumer listens on port LTI_PORT. This 
    constant is a class variable; change to taste.
    
    Expected format from LTI consumer is:
         {
            "ltiKey"         : <lti-key>,
            "ltiSecret"      : <lti-secret>,
            "action"      : {"publish" | "subscribe" | "unsubscribe"},
            "bus_topic"   : <schoolbus topic>>
            "payload"     :
            {
            "course_id": course_id,
            "resource_id": problem_id,
            "student_id": anonymous_id_for_user(user, None),
            "answers": answers,
            "result": is_correct,
            "event_type": event_type,
            "target_topic" : schoolbus_topic
            }
        }
        
     An example payload for action publish:

        {
            "ltiKey" : "myLtiKey",
            "ltiSecret" : "myLtiSecret",
            "action" : "publish,
            "bus_topic" : "studentAction",
            "payload" :
		    {"event_type": "problem_check",
		      "resource_id": "i4x://HumanitiesSciences/NCP-101/problem/__61",
		      "student_id": "d4dfbbce6c4e9c8a0e036fb4049c0ba3",
		      "answers": {"i4x-HumanitiesSciences-NCP-101-problem-_61_2_1": ["choice_3", "choice_4"]},
		      "result": False,
		      "course_id": "HumanitiesSciences/NCP-101/OnGoing",
            }
        }
        
    Example for subscribe:
        {
            "ltiKey" : "myLtiKey",
            "ltiSecret" : "myLtiSecret",
            "action" : "publish,
            "bus_topic" : "studentAction",
            "payload" : {
		                  "delivery_url" : "https://myMachine.myDomain.edu"
		                }
        }
        
    When a message arrives on the bus, it will be
    POSTed to each subscribed URL, with this body:
    
        {
            "ltiKey" : "myLTIKey",
            "ltiSecret" : "myLTISecret",
            "time" : "2007-01-25T12:00:00Z",
            "payload": "..."
        }
    
    
        
    Authentication is controlled by a config file. See file ltibridge.cnf.example
    of this distribution for the format of this file.
    
    HTTP Error Codes Used:
       400  (Bad Request) if no topic was provided in the request,
                           or if no payload is included,
                           or no delivery URL is included in a subscribe request
       401  (Unauthorized) if ltiKey/ltiSecret are missing or incorrect,
                           or if LTI client is not authorized to subscribe
                           to a particular topic. 
       403  (Forbidden)    if LTI client tries to use HTTP instead of HTTPS
       405  (Method not Allowed) if 'action' field is missing.
                       
       409 (Conflict)      cannot provide both: a POST body, and a GET query
                           in the URL.
       415 (Unsupported Media Type) If message is not legal JSON.
       
       501 (Not Implemented) if 'action' field contains an unknown command.
       
    
    To test, you can use https://www.hurl.it/ with URL: 
       https://yourServer.edu:7075/schoolbus 
    replacing 7075 with the value of LTI_PORT in your installation.
    
    '''

    LTI_BRIDGE_SERVICE_PORT = 7075
    
    # Time to wait for LTI provider (e.g. LMS) to
    # respond when trying to deliver a bus message to it:
    LTI_BRIDGE_DELIVERY_TIMEOUT = 1 # second

    # Remember whether logging has been initialized (class var!):
    loggingInitialized = False
    logger = None
    
    # Path to config file, which holds LTI keys/secrets:
    configfile = None
    # Config file's modification time when auth_dict
    # is initialized from the file:
    auth_file_mod_time = None
    
    # Dict with info obtained from the authentication 
    # config file:
    auth_dict = {}
    
    # Whether or not the redis-server was running when this bridge
    # service was started. If it wasn't running, we start it as
    # part of our startup. The following remembers whether we
    # did start, so we have a choice of killing it upon exit:
    redis_pid = None     
    
    # Keep track of SchoolBus subscriptions:
    
    # File in which jsonfiledict will store subscriptions:
    subscriptions_path = os.path.join(os.path.dirname(__file__), '../../subscriptions/lti_bus_subscriptions.json')

    def initialize(self):
        '''
        This method is call once when the server is started. In 
        contrast, the __init__() method, which we don't override
        in this class is called every time a new request arrives.
        '''
        
        # Create a BusAdapter instance that handles all
        # interactions with the SchoolBus:
        self.busAdapter = BusAdapter()
        
        # Create or read existing JSON file with all
        # subscriptions:
        try:
            self.lti_subscriptions = JsonFileDict(LTISchoolbusBridge.subscriptions_path)
            self.lti_subscriptions.load()
        except (ValueError, IOError):
            # The persistent-subscription file was absent,
            # or contained non-JSON:
            try:
                with open(LTISchoolbusBridge.subscriptions_path, 'r') as fd:
                    subscriptions_raw = fd.readlines()
                if subscriptions_raw is not None and len(subscriptions_raw) > 0:
                    self.logErr('Bad JSON in subscription file %s: %s' % (LTISchoolbusBridge.subscriptions_path,
                                                                          str(subscriptions_raw)))
            except Exception:
                # Can't even read the subscription file:
                self.logErr('Could not read subscription file %s' % LTISchoolbusBridge.subscriptions_path)
            with open(LTISchoolbusBridge.subscriptions_path, 'w') as fd:
                fd.write('{}')
            self.lti_subscriptions = {}
        self.logInfo('Loaded existing subscriptions: %s' %\
                      str(self.lti_subscriptions) if len(self.lti_subscriptions) > 0 else 'No subscriptions on record.')
        
        # Callback for BusAdapter when a message arrives
        # on the bus, destined for an LTI end point:
        self.bus_in_msg_callback = functools.partial(self.bus_to_lti_callback)
        # Bus-inmsg-handler:
        self.bus_in_msg_handler  = functools.partial(self.to_lti_transmitter)
        
        self.published_to_bus_counter = 0
        self.delivered_to_lti_counter = 0
        
        # If there are subscriptions from last time this
        # server ran, then re-subscribe to them:
        for bus_topic in self.lti_subscriptions.keys():
            self.logInfo('Subscribing to bus topic %s' % bus_topic)
            self.busAdapter.subscribeToTopic(bus_topic, self.bus_in_msg_callback)
        
    # -------------------------------- HTTP Handler ---------

    def post(self):
        '''
        Override the post() method. The
        associated form is available as a 
        dict in self.request.arguments.
        
        Logs errors: Bad json in the POST body, missing SchoolBus topic, missing payload. 
        
        '''
        postBodyForm = self.request.body
        #print(str(postBody))
        #self.write('<!DOCTYPE html><html><body><script>document.getElementById("ltiFrame-i4x-DavidU-DC1-lti-2edb4bca1198435cbaae29e8865b4d54").innerHTML = "Hello iFrame!"</script></body></html>"');    

        #self.echoParmsToEventDispatcher(postBodyForm)
        
        try:
            # Turn POST body JSON into a dict:
            postBodyDict = json.loads(str(postBodyForm))
        except ValueError:
            self.logErr('POST called with improper JSON: %s' % str(postBodyForm))            
            self.returnHTTPError(415, 'Message did not include a proper JSON object %s' % str(postBodyForm))
            return

        # Does msg contain the required 'action' field?
        action = postBodyDict.get('action', None)
        if action is None:
            self.logErr("POST called without action field: '%s'" % str(postBodyDict))
            self.returnHTTPError(405, 'Message did not include an action field: %s' % str(postBodyDict))
            return
        # Normalize capitalization:
        action = action.lower()
                
        # Is the required bus_topic field present?                
        target_topic = postBodyDict.get('bus_topic', None)
        if target_topic is None:
            self.logErr('POST called without target_topic specification: %s' % str(postBodyDict))
            self.returnHTTPError(400, 'Message did not include a target_topic field: %s' % str(postBodyDict))
            return

        # Look for LTI key and secret in the dict, and
        # check it against the config file:

        if not self.check_auth(postBodyDict, target_topic):
            # check_auth will return the correct HTTP Error
            # before returning.
            return
            
        payload = postBodyDict.get('payload', None)
        if payload is None:
            self.logErr('POST called without payload field: %s' % str(postBodyDict))
            self.returnHTTPError(400, 'Message did not include a payload field: %s' % str(postBodyDict))
            return

        # Ensure that payload is good JSON;
        if not isinstance(payload, dict):
            try:
                json.loads(payload)
            except ValueError:
                self.logDebug('Bad JSON in payload field of message: %s' % str(payload))
                self.returnHTTPError(415, 'Message payload field does not contain proper JSON: %s' % str(postBodyDict))
                return
        
        # Finally, seems to be a legal msg; process the various actions:
        if action == 'publish':
            self.logDebug("Req to publish to '%s': %s" % (target_topic, str(payload)))
            self.publish_to_bus(target_topic, payload)
            return
        elif action in ['subscribe', 'unsubscribe']:
            # Must have a URL in the payload:
            delivery_url = payload.get('delivery_url', None)
            if delivery_url is None:
                self.logErr("POST called with action '%s', but no delivery URL provided: %s" % (action, str(postBodyDict)))
                self.returnHTTPError(400, "Action '%s' must provide a delivery_url in the payload field; offending message: '%s'" % (action, str(postBodyDict)))
                return
            # Do minimal check of the URL: must be scheme HTTPS to 
            # ensure that message from the bus to the delivery URL are
            # encrypted. Since the delivery will be a POST, there shouldn't
            # be a query or fragment part:
            url_segments = urlparse.urlparse(delivery_url)
            if url_segments.scheme.lower() != 'https':
                self.logErr("POST request specifying non-secure URL '%s': '%s'" % (delivery_url, str(postBodyDict)))
                self.returnHTTPError(403, "Delivery URL must use an encrypted scheme (https); was %s. Offending POST body '%s'" % (delivery_url, str(postBodyDict)))
                return
            if len(url_segments.query) + len(url_segments.fragment) > 0:
                self.logErr("POST request with non-empty query or fragment URL: '%s'" % str(postBodyDict))
                self.returnHTTPError(409, "Delivery URL must not have a query or fragment part, but was '%s'. Offending POST body '%s'" % (delivery_url, str(postBodyDict)))
                return
            # Finally, all seems good for subscribe/unsubsribe:
            if action == 'subscribe':
                self.logInfo('Subscribing to %s; LTI client: %s' % (target_topic, delivery_url))
                self.lti_subscribe(target_topic, delivery_url)
            else:
                self.logInfo('Unsubscribing from %s; LTI client: %s' % (target_topic, delivery_url))
                self.lti_unsubscribe(target_topic, delivery_url)
            return
        else:
            # Unknown action:
            self.logErr("POST called with unknown action value '%s': '%s'" % (action, str(postBodyDict)))
            self.returnHTTPError(501, "Action '%s' is not implemented; offending message: '%s'" % (action, str(postBodyDict)))
            return
            
        return
            
        
    def check_auth(self, postBodyDict, target_topic):
        '''
        Given the payload dictionary and the SchoolBus topic to
        which the information is to be published, check authentication.
        Return True if authentication checks out, else return False.
        If authentication fails for any reason, an appropriate HTTP response
        header will have been sent. The caller should simply abandon
        the request for which authentication was being checked.
        
        The method expectes LTISchoolbusBridge.auth_dict to be initialized.
        If the configuration file that underlies the dict does not have
        an entry for the given topic, auth fails. If the LTI key or LTI secret
        are absent from postBodyDict, auth fails. If either secret or key
        in the payload does not match the key/secret in the config file,
        auth fails. See class comment for config file format.
        
        :param postBodyDict: dictionary parsed from payload JSON
        :type postBodyDict: {string : string}
        :param target_topic: SchoolBus topic for which authentication is to be checked
        :type target_topic: str
        '''
        
        try:
            given_key = postBodyDict['ltiKey']
            given_secret = postBodyDict['ltiSecret']
        except KeyError:
            self.logErr('Either key or secret missing in incoming POST: %s' % str(postBodyDict))
            self.returnHTTPError(401, 'Either key or secret were not included in LTI request: %s' % str(postBodyDict))
            return False
        except TypeError:
            self.logErr('POST body of LTI request did not parse into a Python dictionary: %s' % str(postBodyDict))
            self.returnHTTPError(415, 'POST body of LTI request did not parse into a Python dictionary: %s' % str(postBodyDict))
            return False
        
        try:
            # Get sub-dict with secret and key from config file
            # See class header for config file format:
            auth_entry = LTISchoolbusBridge.auth_dict[target_topic]

            # Compare given key and secret with the key/secret on file
            # for the target bus topic:            
            key_on_record = auth_entry['ltiKey'] 
            if key_on_record != given_key:
                # One more chance: was auth file updated since we
                # last loaded it into auth_dict?
                if self.reload_auth_if_new():
                    # Load the modified auth info again:
                    auth_entry = LTISchoolbusBridge.auth_dict[target_topic]
                    key_on_record = auth_entry['ltiKey'] 
                    # Now is all OK?
                    if key_on_record != given_key:
                        self.logErr("Key '%s' does not match key for topic '%s' in config file." % (given_key, target_topic))
                        # Required response header field for 401-not authenticated:
                        self.set_header('WWW-Authenticate', 'key/secret')
                        self.returnHTTPError(401, "Service not authorized for bus topic '%s'" % target_topic)
                        return False
                else:
                    # Bad ltiKey:
                    self.logErr("Key '%s' does not match key for topic '%s' in config file." % (given_key, target_topic))
                    # Required response header field for 401-not authenticated:
                    self.set_header('WWW-Authenticate', 'key/secret')
                    self.returnHTTPError(401, "Service not authorized for bus topic '%s'" % target_topic)
                    return False
                    
                    
            secret_on_record = auth_entry['ltiSecret'] 
            if secret_on_record != given_secret:
                # One more chance: was auth file updated since we
                # last loaded it into auth_dict?
                if self.reload_auth_if_new():
                    # Load the modified auth info again:
                    auth_entry = LTISchoolbusBridge.auth_dict[target_topic]
                    secret_on_record = auth_entry['ltiSecret'] 
                    # Now is all OK?
                    if secret_on_record != given_secret:
                        self.logErr("Secret '%s' does not match secret for topic '%s' in config file." % (given_secret, target_topic))
                        # Required response header field for 401-not authenticated:
                        self.set_header('WWW-Authenticate', 'key/secret')
                        self.returnHTTPError(401, "Service not authorized for bus topic '%s'" % target_topic)
                        return False
                else:
                    self.logErr("Secret '%s' does not match secret for topic '%s' in config file." % (given_secret, target_topic))
                    # Required response header field for 401-not authenticated:
                    self.set_header('WWW-Authenticate', 'key/secret')
                    self.returnHTTPError(401, "Service not authorized for bus topic '%s'" % target_topic)
                    return False
                    
                
        except KeyError:
            # Either no config file entry for target topic, or malformed
            # config file that does not include both 'ltikey' and 'ltisecret'
            # JSON fields for given target topic: 
            self.logErr("No entry for topic '%s' in config file, or ill-formed config file; --> Requestor not authorized for this topic" %\
                        target_topic)
            # Required response header field for 401-not authenticated:
            self.set_header('WWW-Authenticate', 'given_key/given_secret')
            self.returnHTTPError(401, "Service not authorized for bus topic '%s'" % target_topic)
            return False

        return True

    def reload_auth_if_new(self):
        '''
        Check whether config file with its LTI keys and secrets
        was modified since last load into auth_dict. If so, update
        auth_dict and return True. Else just return False.
        :return True if auth_dict was updated, else return False.
        :rtype bool
        '''
        
        statinfo = os.stat(LTISchoolbusBridge.configfile)
        if LTISchoolbusBridge.auth_file_mod_time < statinfo.st_mtime and\
           LTISchoolbusBridge.load_auth_info(LTISchoolbusBridge.configfile, except_on_failure=False):
            self.logInfo('Noticed that config file %s changed.' % LTISchoolbusBridge.configfile)
            return True
        else:
            return False
        

    def returnHTTPError(self, status_code, msg):
        '''
        Tells tornado that an error occurred in the processing of a
        POST or GET request.
        
        :param status_code: HTTP return code
        :type status_code: int
        :param msg: Arbitrary message that will appear in the browser's window (i.e. not in the header).
                    Therefore may contain newlines or any other chars.
        :type msg: str
        '''
        self.clear()
        # Seems like newlines are bad after all:
        msg = msg.replace('\n', '<newline>')
        self.set_status(status_code, reason=msg)
        

        # The following, while simple, tries to put msg into the
        # HTTP header, where newlines are illegal. This limitation
        # often prevents return of a faulty JSON structure: 
        # raise tornado.web.HTTPError(status_code=status_code, reason=msg)

    def echoParmsToEventDispatcher(self, postBodyDict):
        '''
        For testing only: Write an HTML form back to the calling browser.
        
        :param postBodyDict: Dict that contains the HTML form attr/val pairs.
        :type postBodyDict: {string : string}
        '''
        paramNames = postBodyDict.keys()
        paramNames.sort()
        self.write('<html><body>')
        self.write('<b>LTI-SchoolBus bridge Was Invoked With Parameters:</b><br><br>')
        for key in paramNames:
            self.write('<b>%s: </b>%s<br>' % (key, postBodyDict[key]))
        self.write("</body></html>")
        
    # -------------------------------- SchoolBus Handler ---------
    
    def publish_to_bus(self, topic, payload):
        '''
        Given a topic and an arbitrary string, publishes the
        string to the SchoolBus.
        
        :param topic: topic to which message will be published
        :type topic: str
        :param payload: will be placed in the bus message content field.
        :type payload: str
        '''
        bus_message = BusMessage(content=payload, topicName=topic)
        self.busAdapter.publish(bus_message)
        
        self.published_to_bus_counter += 1
        # Note every 100 messages:
        if self.published_to_bus_counter % 100 == 0:
            self.logInfo('Published total of %s messages to bus.' % self.published_to_bus_counter)
    
    def lti_subscribe(self, topic, url):
        '''
        Allows LTI consumers to subscribe to SchoolBus topics. 
        The consumer must supply a URL to which arriving messages
        and their time stamps are POSTed. It is legal to subscribe
        to the same topic multiple times with different URLs. All 
        URLs will be POSTed to with incoming messages. It is safe
        to subscribe to the same topic with the same URL multiple
        times. This situation is a no-op. It is also legal to have message
        of multiple topics delivered to the same consumer URL.
        
        :param topic: the SchoolBus topic to listen to
        :type topic: str
        :param url: URI where consumer is ready to receive POSTs with incoming messages
        :type url: str
        '''

        try:
            # Do we already have this URL subscribed for this topic?
            self.lti_subscriptions[topic].index(url)
        except KeyError:
            # Nobody is currently subscribed to the topic:
            self.lti_subscriptions[topic] = [url]
            self.lti_subscriptions.save()
        except ValueError:
            # There are subscriptions to the topic, but url is not among them;
            # this is the 'normal' case:
            self.lti_subscriptions[topic].append(url)
            self.lti_subscriptions.save()

        self.busAdapter.subscribeToTopic(topic, functools.partial(self.to_lti_transmitter))
                
    def lti_unsubscribe(self, topic, url):
        '''
        Allows LTI consumers to unsubscribe from a SchoolBus topic.
        It is safe to unsubscribe from a topic/url without first
        subscribing. This event is a no-op. If the given topic
        is subscribed to with multiple delivery URLs, only the
        given URL will no longer receive messages on that topic.
        
        :param topic: topic from which to unsubscribe
        :type topic: str
        :param url: delivery URI associated with the topic 
        :type url: str
        '''
        self.busAdapter.unsubscribeFromTopic(topic)
        try:
            self.lti_subscriptions[topic].remove(url)
            self.lti_subscriptions.save()
        except (KeyError, ValueError):
            # Subscription wasn't in our records:
            pass
        
    def bus_to_lti_callback(self, bus_msg):
        '''
        Called from BusAdapter when a bus message arrives
        for one or more LTI components. We just schedule
        the real handler with the ioloop. This is so that
        the Tornado and BusAdapter threads don't interfere:
        the IOLoop is not thread-safe. self.bus_in_msg_handler
        is the partial function for method to_lti_transmitter()
        
        :param bus_msg: message that arrived on the bus 
        :type bus_msg: BusMessage
        '''
        tornado.ioloop.IOLoop.current().add_callback(self.bus_in_msg_handler, bus_msg)
        
    def to_lti_transmitter(self, bus_msg):
        '''
        Called by BusAdapter with incoming messages to which at least
        one LTI consumer has subscribed. Delivers the message to
        all URLs that were provided in previous calls to lti_subscribe().
        Delivery will be JSON:
            {
                "time"   : "ISO time string",
                "topic"  : "SchoolBus topic of bus message",
                "payload": "message's 'content' field"
            }
        Logged errors: 
             - no subscribers for topic: unsubscribes from the topic as side effect
             - URL is not reachable, so POST failed
             - HTTP-based error returned during POST
        
        :param bus_msg: the incoming SchoolBus message
        :type bus_msg: BusMessage
        '''
        topic = bus_msg.topicName
        try:
            # Get the list of LTI URLs where msgs of this topic are to
            # be delivered:
            subscriber_urls = self.lti_subscriptions[topic]
        except KeyError:
            self.logErr("Server received msg for topic '%s', but subscriber dict has no subscribers for that topic." % topic)
            self.busAdapter.unsubscribeFromTopic(topic)
            return
        
        # Look up the ltiKey and ltiSecret for the
        # topic:
        # Get sub-dict with secret and key from config file
        # See class header for config file format:
        try:
            auth_entry = LTISchoolbusBridge.auth_dict[topic]
            (ltiKey, ltiSecret) = (auth_entry['ltiKey'], auth_entry['ltiSecret'])
        except KeyError:
            # Yes, there is a subscriber for this topic, but
            # not a key and/or secret.
            self.logErr('Received bus msg on topic %s to which subscriptions existed, but no key/secret.' % topic)
            # Unsubscribe from this topic:
            self.busAdapter.unsubscribeFromTopic(topic)
            return
        
        msg_to_post = '{"time" : "%s", "ltiKey" : "%s", "ltiSecret" : "%s", "bus_topic" : "%s", "payload" : "%s"}' %\
            (bus_msg.isoTime, ltiKey, ltiSecret, topic, bus_msg.content)

        # POST the msg to each LTI URL that requested the topic:
        for lti_subscriber_url in subscriber_urls:
            try:
                request = urllib2.Request(lti_subscriber_url, msg_to_post, {'Content-Type': 'application/json'})
                response = urllib2.urlopen(request,             #@UnusedVariable
                                           json.dumps(msg_to_post),
                                           timeout=LTISchoolbusBridge.LTI_BRIDGE_DELIVERY_TIMEOUT) 

#****                #r = requests.post(lti_subscriber_url, data=msg_to_post, verify=False)
#                 r = requests.post(lti_subscriber_url, 
#                                   data=msg_to_post, 
#                                   cert=['/home/paepcke/.ssl/duo_stanford_edu.pem',
#                                         '/home/paepcke/.ssl/duo.stanford.edu.key'],
#                                   verify=True)
            except URLError as e:
                self.logErr('Bad delivery URL %s, SSL configuration for topic %s, or server down (%s).' %\
                             (lti_subscriber_url, topic, `e`))
                continue
#            (status, reason) = (r.status_code, r.reason)
#            if status != 200:
#                self.logErr("Failed to deliver bus message to subscriber %s; %s: %s" % (lti_subscriber_url, status, reason))
            self.delivered_to_lti_counter += 1
            # Note every 100 deliveries:
            if self.delivered_to_lti_counter % 100 == 0:
                self.logInfo('Delivered total of %s messages to LTI clients.' % self.delivered_to_lti_counter)
            
            
    # -------------------------------- Utilities ---------            
        
    @classmethod
    def setupLogging(cls, loggingLevel, logFile=None):
        if cls.loggingInitialized:
            # Remove previous file or console handlers,
            # else we get logging output doubled:
            cls.logger.handlers = []
            
        # Set up logging:
        cls.logger = logging.getLogger('ltibridge')
        if logFile is None:
            handler = logging.StreamHandler()
        else:
            handler = TimedRotatingFileHandler(filename=logFile, 
                                               when='D',          # Units in days
                                               interval=30,       # Rotate every 30 days
                                               backupCount=6)     # Keep at most 6 old logs
        formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
        handler.setFormatter(formatter)            
        cls.logger.addHandler(handler)
        cls.logger.setLevel(loggingLevel)
        cls.loggingInitialized = True
 
    def logDebug(self, msg):
        LTISchoolbusBridge.logger.debug(msg)

    def logWarn(self, msg):
        LTISchoolbusBridge.logger.warn(msg)

    def logInfo(self, msg):
        LTISchoolbusBridge.logger.info(msg)

    def logErr(self, msg):
        LTISchoolbusBridge.logger.error(msg)
        
    @classmethod
    def load_auth_info(cls, configfile, except_on_failure=False):
        '''
        Given location of the configuration file, which holds
        the keys and secrets of authorized LTI components,
        (re)-initialize the class variable auth_dict from
        that file. It is ok to call this method more than once.
        Class variable LTISchoolbusBridge.auth_file_mod_time is
        updated with current file mod time.
        
        :param configfile: full path to config file; by default in $HOME/.ssh/ltibridge.cnf.
        :type configifle: string
        :param except_on_failure: If True exceptions are raised if 
            file not available, or content is ill-formed. If False
            then boolean return value signifies successful load.
        :return True if auth info could be loaded and parsed. Else False.
        :rtype bool
        :raised IOError
        :raised ValueError 
        '''

        try:
            with open(configfile, 'r') as conf_fd:
                # Use jsmin to remove any C/C++ comments from the
                # config file:
                LTISchoolbusBridge.auth_dict = json.loads(jsmin(conf_fd.read()))
                statinfo = os.stat(configfile)
                LTISchoolbusBridge.auth_file_mod_time = statinfo.st_mtime
        except IOError:
            if except_on_failure:
                raise
            else:
                return False
        except ValueError:
            if except_on_failure:
                raise
            else:
                return False

    @classmethod  
    def makeApp(cls, init_parm_dict):
        '''
        Create the tornado application, making it 
        called via http://myServer.stanford.edu:<port>/schoolbus
        
        :param init_parm_dict: keyword args to pass to initialize() method.
        :type init_parm_dict: {string : <any>}
        '''
        
        settings = {
                    'path': os.path.join(os.path.dirname(__file__), 'static_html'),
                    'default_filename': 'index.html'
                    }
        
        # React to HTTPS://<server>:<post>/:  Only GET will work, and will show instructions.
        # and to   HTTPS://<server>:<post>/schoolbus  Only POST will work there.
        handlers = [
                    (r"/schoolbus", LTISchoolbusBridge),
                    (r"/(.*)", tornado.web.StaticFileHandler, settings)
                    ]        
        
        application = tornado.web.Application(handlers)
        return application
    
    @classmethod
    def guess_key_path(cls):
        '''
        Check whether an SSL key file exists, and is readable
        at $HOME/.ssl/<fqdn>.key. If so, the full path is
        returned, else throws IOERROR.

        :raise IOError if default keyfile is not present, or not readable.
        '''
        
        ssl_root = os.getenv('HOME') + '/.ssl'
        fqdn = socket.getfqdn()
        keypath = os.path.join(ssl_root, fqdn + '.key')
        try:
            with open(keypath, 'r'):
                pass
        except IOError:
            raise IOError('No key file %s exists.' % keypath)
        return keypath


    @classmethod
    def guess_cert_path(cls):
        '''
        Check whether an SSL cert file exists, and is readable.
        Will check three possibilities:
        
           - $HOME/.ssl/my_server_edu_cert.cer
           - $HOME/.ssl/my_server_edu.cer
           - $HOME/.ssl/my_server_edu.pem
           
        in that order. 'my_server_edu' is the fully qualified
        domain name of this server.

        If one readable file of that name is found, the full path is
        returned, else throws IOERROR.

        :raise IOError if default keyfile is not present, or not readable.
        '''
        
        ssl_root = os.getenv('HOME') + '/.ssl'
        fqdn = socket.getfqdn().replace('.', '_')
        certpath1 = os.path.join(ssl_root, fqdn + '_cert.cer')
        try:
            with open(certpath1, 'r'):
                return certpath1
        except IOError:
            pass
        try:
            certpath2 = os.path.join(ssl_root, fqdn + '.cer')
            with open(certpath2, 'r'):
                return certpath2
        except IOError:
            pass
        
        certpath3 = os.path.join(ssl_root, fqdn + '.pem')
        try:
            with open(certpath3, 'r'):
                return certpath3
        except IOError:
            raise IOError('None of %s, %s, or %s exists or is readable.' %\
                          (certpath1, certpath2, certpath3))
Exemplo n.º 28
0
class RedisPerformanceTester(object):
    '''
    classdocs
    '''

    PUBLISH_TOPIC = 'test'

    def __init__(self):
        '''
        Constructor
        '''
        self.host = 'localhost'
        self.port = 6379
        
        self.letterChoice = string.letters
        self.bus = BusAdapter()
        
    def close(self):
        self.bus.close()
    
    def publishToUnsubscribedTopic(self, numMsgs, msgLen, block=True):
        '''
        Publish given number of messages to a topic that
        nobody is subscribed to. Each msg is of length msgLen.
        
        :param numMsgs:
        :type numMsgs:
        :param msgLen:
        :type msgLen:
        '''
        (msg, md5) = self.createMessage(msgLen) #@UnusedVariable
        busMsg = BusMessage(content=msg, topicName=RedisPerformanceTester.PUBLISH_TOPIC)
        startTime = time.time()
        for _ in range(numMsgs):
            self.bus.publish(busMsg, block=block)
        endTime = time.time()
        self.printResult('Publishing %s msgs to empty space (block==%s): ' % (str(numMsgs),str(block)), startTime, endTime, numMsgs)
    
    def publishToSubscribedTopic(self, numMsgs, msgLen, block=True, sameProcessListener=True):
        '''
        Publish given number of messages to a topic that
        a thread in this same process is subscribed to.
        These are asynchronous publish() calls. The receiving
        thread computes MD5 of the received msg to verify.
        
        :param numMsgs:
        :type numMsgs:
        :param msgLen:
        :type msgLen:
        :param block: if True, publish() will wait for server confirmation.
        :type block: bool
        :param sameProcessListener: if True, the listener to messages will be a thread in this
            Python process (see below). Else an outside process is expected to be subscribed
            to RedisPerformanceTester.PUBLISH_TOPIC.
        :type sameProcessListener: bool
        '''

        (msg, md5) = self.createMessage(msgLen)
        busMsg = BusMessage(content=msg, topicName=RedisPerformanceTester.PUBLISH_TOPIC)
        try:
            listenerThread = ReceptionTester(msgMd5=md5, beSynchronous=False)
            listenerThread.daemon = True
            listenerThread.start()
            
            startTime = time.time()
            for _ in range(numMsgs):
                self.bus.publish(busMsg, block=block)
            endTime = time.time()
            self.printResult('Publishing %s msgs to a subscribed topic (block==%s): ' % (str(numMsgs), str(block)), startTime, endTime, numMsgs)
        except Exception:
            raise
        finally:
            listenerThread.stop()
            listenerThread.join(3)
            
    def syncPublishing(self, numMsgs, msgLen, block=True):

        #sys.stdout.write('Run python src/redis_bus_python/test/test_harness_server.py echo and hit ENTER...')
        #sys.stdin.readline()
        
        (msg, md5) = self.createMessage(msgLen) #@UnusedVariable

        busMsg = BusMessage(content=msg, topicName=ECHO_TOPIC)


        startTime = time.time()
        try:
            for serialNum in range(numMsgs):
                try:
                    busMsg.id = serialNum
                    res = self.bus.publish(busMsg, sync=True, timeout=5, block=block) #@UnusedVariable
                except SyncCallTimedOut:
                    #printThreadTraces()
                    raise
                    
            endTime = time.time()
            self.printResult('Publishing %s synch msgs (block==%s): ' % (str(numMsgs), str(block)), startTime, endTime, numMsgs)
        finally:
            pass
        
        
    def rawIronPublish(self, numMsgs, msgLen, block=True):
        
        sock = self.bus.topicWaiterThread.pubsub.connection._sock
        sock.setblocking(1)
        sock.settimeout(2)
        
        (msg, md5) = self.createMessage(msgLen) #@UnusedVariable
        wireMsg = '*3\r\n$7\r\nPUBLISH\r\n$4\r\ntest\r\n$190\r\n{"content": \r\n "dRkLSUQxFVSHuVnEekLtfPsXULtWEESQwaRYZtxpFGYRGphNTkRQAMPJfDxoGKOKPCMmptZBriVVfV\r\n LvYisehirsYSHdDrhXRgGl", "id": "a62b8cde-6bf6-4f75-a1f3-e768bec4d5e1", "time": 1437516321946}\r\n'
        num_sent = 0
        start_time = time.time()
        try:
            for _ in range(numMsgs):
                sock.sendall(wireMsg)
                num_sent += 1
    #             if num_sent % 1000 == 0:
    #                 print('Sent %d' % num_sent)
                if block:
                    #time.sleep(0.01)
                    numListeners = sock.recv(1024) #@UnusedVariable
        except socket.timeout:
            end_time = time.time()
            self.printResult('Sending on raw socket; result timeout after %d msgs.' % num_sent, start_time, end_time, numMsgs)
            sys.exit()  
        except Exception:
            end_time = time.time()
            self.printResult('Sending on raw socket; error %d msgs.' % num_sent, start_time, end_time, numMsgs)
            raise
            
        end_time = time.time()
        self.printResult('Sent %d msgs on raw socket; block==%s' % (numMsgs, block), start_time, end_time, numMsgs)
    
    def justListenAndCount(self):
        listenerThread = ReceptionTester(beSynchronous=False)
        listenerThread.daemon = True
        listenerThread.start()
        signal.pause()
        listenerThread.stop()
        listenerThread.join()
    
    def createMessage(self, msgLen):
        '''
        Returns a string of a given length, and its checksum
        
        :param msgLen: desired str length
        :type msgLen: int
        '''
        
        msg = bytearray()
        for _ in range(msgLen):
            msg.append(random.choice(self.letterChoice))
        return (str(msg), hashlib.md5(str(msg)).hexdigest())

    def _connect(self):

        err = None
        # Get addr options for Redis host/port, with arbitrary
        # socket family (the 0), and stream type:
        for res in socket.getaddrinfo(self.host, self.port, 0,
                                      socket.SOCK_STREAM):
            family, socktype, proto, canonname, socket_address = res  #@UnusedVariable
            sock = None
            try:
                sock = socket.socket(family, socktype, proto)
                # TCP_NODELAY
                sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
                
                sock.connect((self.host, self.port))                
                return sock

            except socket.error as _:
                err = _
                if sock is not None:
                    sock.close()

        if err is not None:
            raise err
        raise socket.error("socket.getaddrinfo returned an empty list")

    
    def printResult(self, headerMsg, startTime, endTime, numMsgs):
        totalTime  = endTime - startTime
        timePerMsg = totalTime/numMsgs
        msgPerSec  = numMsgs/totalTime
        print(headerMsg)
        print('msgsPerSec: %s' % str(msgPerSec))
        print('timePerMsg: %s' % str(timePerMsg))
Exemplo n.º 29
0
class MessageOutStreamer(threading.Thread):
    '''
    Thread that keeps sending the same message over and over,
    except for changing the time field.
    '''
    def __init__(self, busMsg=None):
        super(MessageOutStreamer, self).__init__()

        self.daemon = True
        self.streamBus = BusAdapter()
        self.busMsg = BusMessage() if busMsg is None else busMsg

        self.done = False
        self._paused = False
        self._stream_interval = STREAM_INTERVAL

    def pause(self, do_pause):
        if do_pause:
            self._paused = True
        else:
            self._paused = False

    @property
    def paused(self):
        return self._paused

    @property
    def stream_interval(self):
        return self._stream_interval

    @stream_interval.setter
    def stream_interval(self, new_val):
        '''
        Note: we trust that caller ensures new_val to 
        be a non-negative float. 
        
        :param new_val: number of (fractional) seconds to wait between
            stream messages. Zero means no wait.
        :type new_val:float
        '''
        self._stream_interval = new_val

    def change_stream_topic(self, newTopic):
        self.busMsg.topicName = newTopic

    def change_stream_content(self, newContent):
        self.busMsg.content = newContent

    def run(self):
        '''
        Keep publishing one message over and over,
        
        :param msgLen: length of payload
        :type msgLen: int
        '''
        while not self.done:
            self.busMsg.time = time.time()
            self.streamBus.publish(self.busMsg)
            while self._paused and not self.done:
                time.sleep(1)
            if self._stream_interval > 0:
                time.sleep(self._stream_interval)

    def stop(self, signum, frame):
        self.done = True
        self.streamBus.close()
Exemplo n.º 30
0
 def setUpClass(cls):
     cls.bus = BusAdapter()
Exemplo n.º 31
0
    (certFile, keyFile) = JsBusBridge.getCertAndKey()
    sslArgsDict = {'certfile': certFile, 'keyfile': keyFile}

    # For SSL:
    if SSL_USED:
        protocol_spec = 'wss'
        http_server = tornado.httpserver.HTTPServer(application,
                                                    ssl_options=sslArgsDict)
        application.listen(JS_BRIDGE_WEBSOCKET_PORT, ssl_options=sslArgsDict)
    else:
        protocol_spec = 'ws'
        http_server = tornado.httpserver.HTTPServer(application)
        application.listen(JS_BRIDGE_WEBSOCKET_PORT)

    JsBusBridge.bus = BusAdapter(host=bus_server)

    start_msg = 'Starting JavaScript/SchoolBus bridge server on %s://%s:%d/jsbusbridge/' % \
        (protocol_spec, socket.gethostname(), JS_BRIDGE_WEBSOCKET_PORT)

    print(start_msg)

    try:
        ioloop = tornado.ioloop.IOLoop.instance()
        ioloop.start()
    except KeyboardInterrupt:
        shutdown()
    except Exception as e:
        print('Bombed out of tornado IO loop: %s' % ` e `)

    print('School bus JavaScript bridge has shut down.')
Exemplo n.º 32
0
class SchoolbusWikipedia(object):
    '''
    {"summary" : "2", "topic" : "Germany"}
    
    '''
    TOPIC = 'wikipedia'

    def __init__(self):
        '''
        Constructor
        '''
        self.bus = BusAdapter()
        self.bus.subscribeToTopic(SchoolbusWikipedia.TOPIC, 
                                  functools.partial(self.get_info_handler))
        
        # Hang till keyboard_interrupt:
        try:
            self.exit_event = threading.Event().wait()
        except KeyboardInterrupt:
            print('Exiting wikipedia module.')
        
    
    def summary(self, term, num_sentences=1):
        return wikipedia.summary(term, sentences=num_sentences)
    
    def page(self, term):
        return wikipedia.search(term, results=1)
    
    def geosearch(self, lat, longitude, num_results=1, radius=1000):
        return wikipedia.geosearch(lat, longitude, results=num_results, radius=radius)
    
    
    def get_info_handler(self, bus_message):
        '''
        {'topic'   : <keyword>,
         'summary' : <numSentences>,
         'geosearch' : {'lat'    : <float>,
                        'long'   : <float>
                        'radius' : <int>
                        },
         'coordinates' : 'True',
         'references'  : 'True'
         }
         
        :param bus_message:
        :type bus_message:
        '''
        #print(bus_message.content)
        try:
            req_dict = json.loads(bus_message.content)
        except ValueError:
            err_resp = {'error' : 'Bad json in wikipedia request: %s' % str(bus_message.content)}
            resp = self.bus.makeResponseMsg(bus_message, 
                                            json.dumps(err_resp))
            self.bus.publish(resp)
            return
        
        try:
            self.check_req_correctness(req_dict)
        except ValueError as e:
            err_resp = {'error' : '%s' % `e`}
            resp = self.bus.makeResponseMsg(bus_message, json.dumps(err_resp))
            self.bus.publish(resp)
            
        res_dict = {}
        
        if req_dict.get('summary', None) is not None:
            summary = wikipedia.summary(req_dict['topic'], 
                                        sentences=req_dict['summary'])
            res_dict['summary'] = summary.encode('UTF-8', 'replace')
            wants_page = False
        else:
            # Wants whole page content:
            wants_page = True
            
        if req_dict.get('geosearch', None) is not None:
            geo_parms = req_dict['geosearch'] 
            lat = geo_parms['lat']
            longitude = geo_parms['long']
            radius = geo_parms['radius']
            
            res_dict['geosearch'] = wikipedia.geosearch(lat, longitude, req_dict['topic'], radius)
        
        # Remaining request possibilities require the page to be obtained,
        # even if summary was requested:
        page = None
        
        if req_dict.get(u'coordinates', None) is not None and req_dict['coordinates']:
            if page is None:
                page = wikipedia.page(req_dict['topic'])
            try:
                (lat, longitude) = page.coordinates
                res_dict['coordinates'] = '{"lat" : "%s", "long" : "%s"}' %\
                                            (str(lat), str(longitude))
            except KeyError:
                # Wikipedia entry has not coordinates associated with it:
                res_dict['coordinates'] = '"None"'
            
        if req_dict.get('references', None) is not None and req_dict['references']:
            if page is None:
                page = wikipedia.page(req_dict['topic'])
            res_dict['references'] = page.coordinates

        if wants_page:
            if page is None:
                page = wikipedia.page(req_dict['topic'])
            res_dict['content'] = page.content
            
        resp_msg = self.bus.makeResponseMsg(bus_message, json.dumps(res_dict))
        self.bus.publish(resp_msg)
                     
    def check_req_correctness(self, req_dict):
        if req_dict.get('topic', None) is None:
            raise ValueError('No topic supplied in wikipedia request.')

        if req_dict.get('summary', None) is not None:
            # Must have numSentences as int:
            num_sentences = req_dict['summary']

            try:
                num_sentences = int(num_sentences)
            except ValueError:
                raise ValueError('Summary request must have a positive integer indicating number of sentences requested; was %s' %\
                                 str(num_sentences))
                                 
            if num_sentences < 1:
                raise ValueError('Summary request must have a positive integer indicating number of sentences requested; was %s' %\
                                  str(num_sentences))
                
        if req_dict.get('geosearch', None) is not None:
            geo_parms = req_dict['geosearch']
            try:
                if type(geo_parms['lat'])  != float or\
                   type(geo_parms['long']) != float or\
                   type(geo_parms['radius']) != int or\
                   geo_parms['radius'] < 1:
                    raise ValueError('Bad parameters to wikipedia geo search (lat/long must be floats; radius must be positive int):' %\
                                     str(geo_parms))
            except KeyError:
                raise ValueError('Missing parameter to wikipedia geo search; must have lat/long/radius')
        
        if req_dict.get('coordinates', None) is not None:
            coords_wanted = str(req_dict['coordinates']).lower()
            if coords_wanted != 'true' and coords_wanted != 'false':
                raise ValueError("Request for wikipedia topic coordinates must be 'true', or 'false', not %s" %\
                                 str(coords_wanted))
            # Normalize value to bool:
            req_dict['coordinates'] = True if coords_wanted == 'true' else False  

        if req_dict.get('references', None) is not None:
            refs_wanted = str(req_dict['references']).lower()
            if refs_wanted != 'true' and refs_wanted != 'false':
                raise ValueError("Request for wikipedia topic reference links must be 'true', or 'false', not %s" %\
                                 str(coords_wanted))
            # Normalize to bool:
            req_dict['references'] = True if refs_wanted == 'true' else False                  
Exemplo n.º 33
0
class MsgDeliveryTest(unittest.TestCase):
    def setUp(self):
        self.msg_server = OnDemandPublisher()
        self.msg_server.start()
        self.deliveryDest = functools.partial(self._deliveryDest)
        self.bus = BusAdapter()
        self.delivery_event = threading.Event()

        # Content and topic for outgoing msgs sent
        # via the OnDemandPublisher instance:

        self.reference_msg_content = 'rTmzntNSQLmXokesBpLmAbPYeysftXnuntfdPKrxMVNUuqVFHFzfcrrSaRssdHuMRhPYjXKYrJjwKcyeYycEzQSkJubTabeSFLRS'
        self.reference_named_topic = 'MyTopic'
        self.reference_pattern_topic = r'MyTopic*'
        self.reference_pattern_obj = re.compile(self.reference_pattern_topic)
        self.reference_pattern_matching_topic = 'MyTopicHurray'
        self.reference_context = 'myContext'

        # Msg with a non-pattern, i.e. fixed-name topic:
        self.reference_bus_msg = BusMessage(
            topicName=self.reference_named_topic,
            content=self.reference_msg_content,
            context=self.reference_context)

        # Msg with a topic that will match self.reference_pattern_topic:
        self.reference_bus_wildcard_msg = BusMessage(
            topicName=self.reference_pattern_matching_topic,
            content=self.reference_msg_content,
            context=self.reference_context)

    def tearDown(self):
        self.msg_server.stop()
        self.msg_server.join()
        self.bus.close()
        self.delivery_event.clear()

    @unittest.skipIf(not TEST_ALL, 'Temporarily disabled')
    def testDeliveryNameNotThreaded(self):
        self.bus.subscribeToTopic(self.reference_named_topic,
                                  self.deliveryDest,
                                  threaded=False,
                                  context=self.reference_context)

        self.delivery_event.clear()
        self.msg_server.sendMessage(self.reference_bus_msg)

        # Wait for message to arrive:
        self.delivery_event.wait()
        # Compare our expected incoming bus message with
        # what we actually got:
        self.assertBusMsgsEqual(self.reference_bus_msg, self.received_bus_msg)

    @unittest.skipIf(not TEST_ALL, 'Temporarily disabled')
    def testDeliveryNameThreaded(self):
        self.bus.subscribeToTopic(self.reference_named_topic,
                                  self.deliveryDest,
                                  threaded=True,
                                  context=self.reference_context)
        self.delivery_event.clear()
        self.msg_server.sendMessage(self.reference_bus_msg)

        # Wait for message to arrive:
        self.delivery_event.wait()

        # Compare our expected incoming bus message with
        # what we actually got:
        self.assertBusMsgsEqual(self.reference_bus_msg, self.received_bus_msg)

    @unittest.skipIf(not TEST_ALL, 'Temporarily disabled')
    def testDeliveryPatternNotThreaded(self):
        # Subscribe to a pattern topic, using an re.Pattern instance:
        self.bus.subscribeToTopic(self.reference_pattern_obj,
                                  self.deliveryDest,
                                  threaded=False,
                                  context=self.reference_context)

        self.delivery_event.clear()
        # Send to a topic that fits the reference pattern ('MyTopic*'):
        self.msg_server.sendMessage(self.reference_bus_wildcard_msg)

        self.delivery_event.wait()

        self.assertBusMsgsEqual(self.reference_bus_wildcard_msg,
                                self.received_bus_msg)

    @unittest.skipIf(not TEST_ALL, 'Temporarily disabled')
    def testDeliveryPatternThreaded(self):
        # Subscribe to a pattern topic, using an re.Pattern instance:
        self.bus.subscribeToTopic(self.reference_pattern_obj,
                                  self.deliveryDest,
                                  threaded=True,
                                  context=self.reference_context)

        self.delivery_event.clear()
        # Send to a topic that fits the reference pattern ('MyTopic*'):
        self.msg_server.sendMessage(self.reference_bus_wildcard_msg)

        self.delivery_event.wait()

        self.assertBusMsgsEqual(self.reference_bus_wildcard_msg,
                                self.received_bus_msg)

    @unittest.skipIf(not TEST_ALL, 'Temporarily disabled')
    def testSyncPublish(self):

        echo_msg = self.reference_bus_msg

        # Change target topic to what the test harness echo server
        # listens for:
        echo_msg.topicName = self.msg_server.ECHO_CHANNEL

        result = self.bus.publish(echo_msg, sync=True)
        self.assertEqual(self.reference_msg_content, result)

    # -------------------------  Service Methods --------------

    def assertBusMsgsEqual(self, expected_msg, actual_msg):
        self.assertEqual(expected_msg.topicName, actual_msg.topicName)
        self.assertEqual(expected_msg.context, actual_msg.context)
        self.assertEqual(expected_msg.content, actual_msg.content)

    def _deliveryDest(self, bus_msg):
        '''
        Receives incoming messages, and places them into
        instance variable received_bus_msg.
        
        :param bus_msg:
        :type bus_msg:
        '''
        #print('BusMessage: context is %s' % bus_msg.context)
        self.received_bus_msg = bus_msg
        self.delivery_event.set()
Exemplo n.º 34
0
class SchoolbusWikipedia(object):
    '''
    {"summary" : "2", "topic" : "Germany"}
    
    '''
    TOPIC = 'wikipedia'

    def __init__(self):
        '''
        Constructor
        '''
        self.bus = BusAdapter()
        self.bus.subscribeToTopic(SchoolbusWikipedia.TOPIC,
                                  functools.partial(self.get_info_handler))

        # Hang till keyboard_interrupt:
        try:
            self.exit_event = threading.Event().wait()
        except KeyboardInterrupt:
            print('Exiting wikipedia module.')

    def summary(self, term, num_sentences=1):
        return wikipedia.summary(term, sentences=num_sentences)

    def page(self, term):
        return wikipedia.search(term, results=1)

    def geosearch(self, lat, longitude, num_results=1, radius=1000):
        return wikipedia.geosearch(lat,
                                   longitude,
                                   results=num_results,
                                   radius=radius)

    def get_info_handler(self, bus_message):
        '''
        {'topic'   : <keyword>,
         'summary' : <numSentences>,
         'geosearch' : {'lat'    : <float>,
                        'long'   : <float>
                        'radius' : <int>
                        },
         'coordinates' : 'True',
         'references'  : 'True'
         }
         
        :param bus_message:
        :type bus_message:
        '''
        #print(bus_message.content)
        try:
            req_dict = json.loads(bus_message.content)
        except ValueError:
            err_resp = {
                'error':
                'Bad json in wikipedia request: %s' % str(bus_message.content)
            }
            resp = self.bus.makeResponseMsg(bus_message, json.dumps(err_resp))
            self.bus.publish(resp)
            return

        try:
            self.check_req_correctness(req_dict)
        except ValueError as e:
            err_resp = {'error': '%s' % ` e `}
            resp = self.bus.makeResponseMsg(bus_message, json.dumps(err_resp))
            self.bus.publish(resp)

        res_dict = {}

        if req_dict.get('summary', None) is not None:
            summary = wikipedia.summary(req_dict['topic'],
                                        sentences=req_dict['summary'])
            res_dict['summary'] = summary.encode('UTF-8', 'replace')
            wants_page = False
        else:
            # Wants whole page content:
            wants_page = True

        if req_dict.get('geosearch', None) is not None:
            geo_parms = req_dict['geosearch']
            lat = geo_parms['lat']
            longitude = geo_parms['long']
            radius = geo_parms['radius']

            res_dict['geosearch'] = wikipedia.geosearch(
                lat, longitude, req_dict['topic'], radius)

        # Remaining request possibilities require the page to be obtained,
        # even if summary was requested:
        page = None

        if req_dict.get(u'coordinates',
                        None) is not None and req_dict['coordinates']:
            if page is None:
                page = wikipedia.page(req_dict['topic'])
            try:
                (lat, longitude) = page.coordinates
                res_dict['coordinates'] = '{"lat" : "%s", "long" : "%s"}' %\
                                            (str(lat), str(longitude))
            except KeyError:
                # Wikipedia entry has not coordinates associated with it:
                res_dict['coordinates'] = '"None"'

        if req_dict.get('references',
                        None) is not None and req_dict['references']:
            if page is None:
                page = wikipedia.page(req_dict['topic'])
            res_dict['references'] = page.coordinates

        if wants_page:
            if page is None:
                page = wikipedia.page(req_dict['topic'])
            res_dict['content'] = page.content

        resp_msg = self.bus.makeResponseMsg(bus_message, json.dumps(res_dict))
        self.bus.publish(resp_msg)

    def check_req_correctness(self, req_dict):
        if req_dict.get('topic', None) is None:
            raise ValueError('No topic supplied in wikipedia request.')

        if req_dict.get('summary', None) is not None:
            # Must have numSentences as int:
            num_sentences = req_dict['summary']

            try:
                num_sentences = int(num_sentences)
            except ValueError:
                raise ValueError('Summary request must have a positive integer indicating number of sentences requested; was %s' %\
                                 str(num_sentences))

            if num_sentences < 1:
                raise ValueError('Summary request must have a positive integer indicating number of sentences requested; was %s' %\
                                  str(num_sentences))

        if req_dict.get('geosearch', None) is not None:
            geo_parms = req_dict['geosearch']
            try:
                if type(geo_parms['lat'])  != float or\
                   type(geo_parms['long']) != float or\
                   type(geo_parms['radius']) != int or\
                   geo_parms['radius'] < 1:
                    raise ValueError('Bad parameters to wikipedia geo search (lat/long must be floats; radius must be positive int):' %\
                                     str(geo_parms))
            except KeyError:
                raise ValueError(
                    'Missing parameter to wikipedia geo search; must have lat/long/radius'
                )

        if req_dict.get('coordinates', None) is not None:
            coords_wanted = str(req_dict['coordinates']).lower()
            if coords_wanted != 'true' and coords_wanted != 'false':
                raise ValueError("Request for wikipedia topic coordinates must be 'true', or 'false', not %s" %\
                                 str(coords_wanted))
            # Normalize value to bool:
            req_dict[
                'coordinates'] = True if coords_wanted == 'true' else False

        if req_dict.get('references', None) is not None:
            refs_wanted = str(req_dict['references']).lower()
            if refs_wanted != 'true' and refs_wanted != 'false':
                raise ValueError("Request for wikipedia topic reference links must be 'true', or 'false', not %s" %\
                                 str(coords_wanted))
            # Normalize to bool:
            req_dict['references'] = True if refs_wanted == 'true' else False
Exemplo n.º 35
0
class OnDemandPublisher(threading.Thread):
    '''
    Server for testing Redis bus modules. Started from the
    command line, or imported into applications. Serves multiple
    functions, individually or together:
    
        1. echoes messages it receives on topic ECHO_TOPIC.
        2. listens to messages without doing anything
        3. sends a continuous stream of messages
        4. check the syntax of an incoming msg, returning the result
        5. send one message on demand.

	USAGE: test_harness_server.py [-h] [-e] [-s [topic [content ...]]] [-c]
	                              [-l topic [topic ...]]
	                              [-o [topic [content ...]]]
	
	optional arguments:
	  -h, --help            show this help message and exit
	  -e, --echo            Echo messages arriving on topic 'echo' as sychronous replies.
	  -s [topic [content ...]], --streamMsgs [topic [content ...]]
	                        Send the same bus message over and over. Topic, or both,
	                        topic and content may be provided. If content is omitted,
	                        a random string of 100 characters will be used.
	                        If topic is omitted as well, messages will be streamed to 'test'
	  -c, --checkSyntax     Check syntax of messages arriving on topic 'bus_syntax'; 
	                        synchronously return result report.
	  -l topic [topic ...], --listenOn topic [topic ...]
	                        Subscribe to given topic(s), and throw the messages away
	  -o [topic [content ...]], --oneshot [topic [content ...]]
	                        Given a topic and a content string, send a bus message. Topic, or both 
	                        topic and content may be provided. If content is omitted,
	                        a random string of 100 characters will be used.
	                        If topic is omitted as well, messages will be streamed to 'test'
	    

    The thread always listens on the ECHO_TOPIC. But
    received messages are only **echoed** if the service was
    started with the 'echo' option
    
    The echo function acts like a synch-call service
    should: echoing on the response topic, which it
    derives from the msg ID. The content will be the
    content of the incoming message, the timestamp
    will be the receipt time.
    
    When echoing, the service keeps track of how many messages 
    it has echoed. But after not receiving any messages for 
    OnDemandPublisher.MAX_IDLE_TIME seconds, the
    current count is printed, and the counter is 
    reset to zero. After every 1000 messages, the total
    number echoed is printed to the console. Every 10,000
    messages, the message/sec rate is printed as well.
    
    When asked to **listen** to some topic, messages on that
    topic are received and counted. Then the message is
    placed into a queue where it can be picked up (self.bus_msg_queue). 
    If nobody attends to the queue, it fills up, and then sits there;
    i.e. no big harm. Statistics printing is as for echoing.
    Stats are placed on the bus_stats_queue for consumption by anyone
    interested (or not).
    
    Messages sent in a **stream** by sendMessageStream()
    contain their content's MD5 in the context field. This constant
    message sending is initiated only if the service is
    started with the 'send_stream' option.

    When asked to check message **syntax**, the service listens
    on SYNTAX_TOPIC, and synchronously returns a string 
    reporting on violations of SchoolBus specifications in 
    the incoming message.
    
    A **oneshot message** is sent if the server is started
    with the --oneshot option. If topic and/or content are provided,
    the message is loaded with the given content and published to
    the given topic. Defaults are to fill the content with a random
    string, and to publish to STREAM_TOPIC.
    
    '''

    # ----------------------------  Constants ----------------

    # Maximum time for no message to arrive before
    # starting over counting messages:
    MAX_IDLE_TIME = 5

    FIND_COMMA_PATTERN = re.compile(r'[,]+')
    FIND_SPACE_PATTERN = re.compile(r'[ ]+')

    LOG_LEVEL_NONE = 0
    LOG_LEVEL_ERR = 1
    LOG_LEVEL_INFO = 2
    LOG_LEVEL_DEBUG = 3

    def __init__(self,
                 serveEchos=True,
                 listenOn=None,
                 streamMsgs=False,
                 checkSyntax=True,
                 oneShotMsg=None):
        '''
        Initialize the test harness server. 
        
        :param serveEchos: if True receives messages on ECHO_TOPIC, and returns  
            message with the same content, but different timestamp as a synchronous call. 
        :type serveEchos: bool
        :param listenOn: if this parameter provides an array of topic(s), messages on these
            topics are received and counted, then placed on self.bus_msg_queue. Intermittent reception counts
            and reception rates are printed to the console, and place on self.bus_stats_queue.
        :type listenOn: {None | [string]}
        :param streamMsgs: if True, a continuous stream of messages are sent to
            STREAM_TOPIC. The timestamp is changed each time.
        :type streamMsgs: bool
        :param checkSyntax: if true, listens on SYNTAX_TOPIC, checks incoming messages
            for conformance to SchoolBus message format, and synchronously returns a result.
        :type checkSyntax: bool
        :param oneShotMsg: if non-none this parameter is expected to be a two-tuple
            containing a topic and a content. Either may be None. If topic is None,
            the message is sent to STREAM_TOPIC. If content is None, content will be
            a random string of length STANDARD_MSG_LENGTH.
        :type oneShotMsg: { None | ({string | None}, {string | None})}
        '''
        threading.Thread.__init__(self)

        self.loglevel = OnDemandPublisher.LOG_LEVEL_DEBUG
        #self.loglevel = OnDemandPublisher.LOG_LEVEL_INFO
        #self.loglevel = OnDemandPublisher.LOG_LEVEL_NONE

        #***********
        #         print ('serveEchos %s' % serveEchos)
        #         print ('listenOn %s' % listenOn)
        #         print ('streamMsgs %s' % streamMsgs)
        #         print('checkSyntax %s' % checkSyntax)
        #         print('oneShotMsg %s' % str(oneShotMsg))
        #         sys.exit()
        #***********

        self._serveEchos = serveEchos

        self.daemon = True
        self.testBus = BusAdapter()
        self.done = False
        # No topics yet that we are to listen to, but drop msgs for:
        self.topics_to_rx = []

        self.echo_topic = ECHO_TOPIC
        self.syntax_check_topic = SYNTAX_TOPIC
        self.standard_msg_len = STANDARD_MSG_LENGTH
        self._stream_interval = STREAM_INTERVAL

        self._stream_topic = STREAM_TOPIC
        self.standard_bus_msg = self.createMessage(STREAM_TOPIC,
                                                   STANDARD_MSG_LENGTH,
                                                   content=None)

        self.one_shot_topic = STREAM_TOPIC
        self.one_shot_content = self.standard_bus_msg

        # Queue into which incoming messages are fed
        # for consumers to communicated to clients, such
        # as a Web UI:
        self.bus_msg_queue = Queue.Queue(MAX_SAVED_MSGS)

        # Queue to which aggregate stats will be written:
        self.bus_stats_queue = Queue.Queue(MAX_SAVED_MSGS)

        # Create a 'standard' message to send out
        # when asked to via SIGUSR1 or the Web:

        if oneShotMsg is not None:
            # The oneShotMsg arg may have one of the following values:
            #   - None if oneshot was not requested.
            #   - An empty array if -o (or --oneshot)
            #        was listed in the command line,
            #        but neither topic nor message content
            #        were specified.
            #   - A one-element array if the topic was
            #        specified
            #   - A two-element array if both topic and
            #        content were specified.

            (one_shot_topic, self.one_shot_content) = oneShotMsg
            if one_shot_topic is not None:
                self.one_shot_topic = one_shot_topic

            # Create a message that one can have sent on demand
            # by sending a SIGUSR1 to the server, or using
            # the Web UI:

            self.standard_oneshot_msg = self.createMessage(
                self.one_shot_topic, content=self.one_shot_content)
            # Send the standard msg:
            self.sendMessage(self.standard_oneshot_msg)
        else:
            # Create a standard message with default topic/content:
            self.standard_oneshot_msg = self.createMessage()

        # Start time of a 10,000 messages batch:
        self.batch_start_time = time.time()

        # Subscribe, and ensure that context is delivered
        # with each message:

        if serveEchos:
            self.serve_echo = True

        if listenOn is not None:
            # Array of topics to listen to
            # If we are to listen to some (additional) topic(s),
            # on which no action is taken, subscribe to it/them now:
            self['topicsToRx'] = listenOn

        if checkSyntax:
            self.check_syntax = True

        self.interruptEvent = threading.Event()

        # Number of messages received and echoed:
        self.numEchoed = 0
        # Alternative to above when not echoing: Number of msgs just received:
        self.numReceived = 0
        self.mostRecentRxTime = None
        self.printedResetting = False

        # Not asked to send a message yet.
        self.sendMsg = False

        if streamMsgs is not None:
            # The streamMsgs arg may have one of the following values:
            #   - None if streaming messages was not requested.
            #   - An empty array if -s (or --streamMsgs)
            #        was listed in the command line,
            #        but neither topic nor message content
            #        were specified.
            #   - A one-element array if the topic was
            #        specified
            #   - A two-element array if both topic and
            #        content were specified.

            self.msg_streamer = MessageOutStreamer(self.standard_bus_msg)

            # The caller passed in a tuple with
            # topic and content for streaming. If they
            # are non-None, update the streamer:

            (stream_topic, stream_content) = streamMsgs
            if stream_topic is not None:
                self.stream_topic = stream_topic
            if stream_content is not None:
                self.stream_content = stream_content

            # Start streaming:

            self.msg_streamer.pause(True)
            self.msg_streamer.start()

        else:
            # Create a standard message with default topic/content:
            self.standard_stream_msg = self.createMessage()
            self.msg_streamer = None

    @property
    def running(self):
        return not self.done

    @property
    def echo_topic(self):
        return self._echo_topic

    @echo_topic.setter
    def echo_topic(self, new_echo_topic):
        if new_echo_topic is None or len(new_echo_topic) == 0:
            # Set to default echo topic:
            new_echo_topic = ECHO_TOPIC
        self._echo_topic = new_echo_topic

    @property
    def serve_echo(self):
        return self._serveEchos

    @serve_echo.setter
    def serve_echo(self, do_serve):
        if do_serve:
            # Have incoming messages delivered to messageReceiver() not
            # via a queue, but by direct call from the library:
            self.testBus.subscribeToTopic(self.echo_topic,
                                          functools.partial(
                                              self.messageReceiver),
                                          threaded=False)
            self._serveEchos = True
        else:
            self.testBus.unsubscribeFromTopic(self.echo_topic)
            self._serveEchos = False

    @property
    def stream_topic(self):
        return self.msg_streamer.busMsg.topicName

    @stream_topic.setter
    def stream_topic(self, new_stream_topic):
        if new_stream_topic is None or len(new_stream_topic) == 0:
            # Set to default stream topic:
            new_stream_topic = STREAM_TOPIC
        self.msg_streamer.change_stream_topic(new_stream_topic)

    @property
    def stream_content(self):
        return self.msg_streamer.busMsg.content

    @stream_content.setter
    def stream_content(self, new_stream_content):
        if new_stream_content is None or len(new_stream_content) == 0:
            self.msg_streamer.change_stream_content(
                self.createRandomStr(self.standard_msg_len))
        else:
            self.msg_streamer.change_stream_content(new_stream_content)

    @property
    def streaming(self):
        return not self.msg_streamer.paused

    @streaming.setter
    def streaming(self, should_stream):
        should_pause = not should_stream
        self.msg_streamer.pause(should_pause)

    @property
    def stream_interval(self):
        return self._stream_interval

    @stream_interval.setter
    def stream_interval(self, new_interval):
        # If the new value is not a number or empty str, set to default:
        if type(new_interval) == str and len(new_interval) == 0:
            new_interval = STREAM_INTERVAL
        else:
            # Non-empty string or number;
            # Ensure float, and replace negative
            # values with 0:
            try:
                if float(new_interval) < 0:
                    new_interval = 0.0
            except ValueError:
                raise ValueError(
                    "Attempt to set streaming interval to a non-numeric quantity: '%s'"
                    % str(new_interval))

        self.msg_streamer.stream_interval = new_interval
        self._stream_interval = new_interval

    @property
    def one_shot_topic(self):
        return self._one_shot_topic

    @one_shot_topic.setter
    def one_shot_topic(self, new_one_shot_topic):
        if new_one_shot_topic is None or len(new_one_shot_topic) == 0:
            # Set to default oneshot topic:
            new_one_shot_topic = STREAM_TOPIC
        else:
            self._one_shot_topic = new_one_shot_topic

    @property
    def one_shot_content(self):
        return self._one_shot_content

    @one_shot_content.setter
    def one_shot_content(self, new_one_shot_content):
        if new_one_shot_content is None or len(new_one_shot_content) == 0:
            self._one_shot_content = self.createRandomStr(
                self.standard_msg_len)
        else:
            self._one_shot_content = new_one_shot_content

    @property
    def syntax_check_topic(self):
        return self._syntax_check_topic

    @syntax_check_topic.setter
    def syntax_check_topic(self, new_syntax_check_topic):
        if new_syntax_check_topic is None or len(new_syntax_check_topic) == 0:
            # Set to default syntax check topic:
            new_syntax_check_topic = SYNTAX_TOPIC
        self._syntax_check_topic = new_syntax_check_topic

    @property
    def check_syntax(self):
        return self._checkSyntax

    @check_syntax.setter
    def check_syntax(self, do_check):
        if do_check:
            self.testBus.subscribeToTopic(
                self.syntax_check_topic,
                functools.partial(self.syntaxCheckReceiver))
            self._checkSyntax = True
        else:
            self.testBus.unsubscribeFromTopic(self.syntax_check_topic)
            self._checkSyntax = False

    @property
    def standard_msg_len(self):
        return self._standard_msg_len

    @standard_msg_len.setter
    def standard_msg_len(self, new_standard_msg_len):
        if new_standard_msg_len == 0 or \
           new_standard_msg_len is None:
            new_standard_msg_len = STANDARD_MSG_LENGTH
        try:
            new_standard_msg_len = int(new_standard_msg_len)
        except:
            new_standard_msg_len = STANDARD_MSG_LENGTH

        self._standard_msg_len = new_standard_msg_len

    def __getitem__(self, item):
        if item == 'echoTopic':
            return self.echo_topic

        elif item == 'echo':
            return self.testBus.subscribedTo(self['echoTopic'])

        elif item == 'streamTopic':
            return self.stream_topic
        elif item == 'streamContent':
            return self.stream_content
        elif item == 'streaming':
            return self.streaming
        elif item == 'streamInterval':
            return self.stream_interval

        elif item == 'oneShotTopic':
            return self.one_shot_topic
        elif item == 'oneShotContent':
            if isinstance(self.one_shot_content, BusMessage):
                return self.one_shot_content.content
            else:
                return self.one_shot_content

        elif item == 'strLen':
            return self.standard_msg_len

        elif item == 'syntaxTopic':
            return self.syntax_check_topic

        elif item == 'chkSyntax':
            return self.testBus.subscribedTo(self['syntaxTopic'])

        elif item == 'topicsToRx':
            return self.topics_to_rx

        else:
            raise KeyError('Key %s is not in schoolbus tester' % item)

    def __setitem__(self, item, new_val):

        if item == 'echoTopic':
            self.echo_topic = new_val

        elif item == 'echo':
            # Deal with both string and bool:
            new_val = (new_val == 'True' or new_val == True)
            self.serve_echo = new_val

        elif item == 'streamTopic':
            self.stream_topic = new_val
        elif item == 'streamContent':
            self.stream_content = new_val
        elif item == 'streaming':
            # Deal with both string and bool:
            new_val = (new_val == 'True' or new_val == True)
            self.streaming = new_val
        elif item == 'streamInterval':
            self.stream_interval = new_val

        elif item == 'oneShotContent':
            self.one_shot_content = new_val

        elif item == 'oneShotTopic':
            self.one_shot_topic = new_val
        elif item == 'oneShotContent':
            self.one_shot_content = new_val

        elif item == 'strLen':
            self.standard_msg_len = new_val

        elif item == 'syntaxTopic':
            self.syntax_check_topic = new_val

        elif item == 'chkSyntax':
            # Deal with both string and bool:
            new_val = (new_val == 'True' or new_val == True)
            self.check_syntax = new_val

        elif item == 'topicsToRx':
            # Topics to which we should listen, but
            # whose msgs we are to receive. Check
            # whether empty str or empty array, which
            # means unsubscribe from all topics-to-rx
            # (though not from syntax checker and echo):
            if ((type(new_val) == str and len(new_val) == 0)) or\
                (type(new_val) == list and len(new_val) == 1 and len(new_val[0]) == 0):
                for (indx, topic) in enumerate(self.topics_to_rx):
                    self.unsubscribeFromTopicOrPattern(topic)
                    del self.topics_to_rx[indx]
                return
            # We are given one or more topics. Tolerate
            # comma-separated, space-separated strings,
            # and also arrays:
            if type(new_val) != list:
                # comma-separated string of topics?
                if OnDemandPublisher.FIND_COMMA_PATTERN.search(
                        new_val) is not None:
                    # Yes, commas found: remove all spaces that might be around the commas:
                    new_val = OnDemandPublisher.FIND_SPACE_PATTERN.sub(
                        '', new_val)
                    new_val = new_val.split(',')
                # Else must be space-separated string of topics or just a single word:
                else:
                    # Replace all multi-space areas with single spaces; the strip()
                    # eliminates spaces at start and end:
                    new_val = OnDemandPublisher.FIND_SPACE_PATTERN.sub(
                        ' ', new_val).strip()
                    # Get an array of topic strings (works even with singleton topic):
                    new_val = new_val.split(' ')

            # Unsubscribe from all topics that are *not*
            # in the new list of topics:
            for (topic_pos, curr_topic) in enumerate(self.topics_to_rx):
                if curr_topic not in new_val:
                    self.topics_to_rx.pop(topic_pos)
                    self.unsubscribeFromTopicOrPattern(curr_topic)

            # Subscribe to any topics in the new-list that
            # we are not already subscribed to:
            for topic in new_val:
                # Disallow use of reserved topics self.echo_topic
                # and self.syntax_check_topic:
                if topic in (self.echo_topic, self.syntax_check_topic):
                    raise ValueError(
                        "Warning: '%s' and '%s' are reserved topic names." %
                        (self.echo_topic, self.syntax_check_topic))

                try:
                    # Already subscribed to it?
                    self.topics_to_rx.index(topic)
                    # At least by our bookkeeping, yes
                    # (else we would have had the exception.
                    # Ensure we really are:
                    if not self.testBus.subscribedTo(topic):
                        self.subscribeTopicOrPattern(topic)
                    continue
                except ValueError:
                    pass
                # New topic:
                self.subscribeTopicOrPattern(topic)
                self.topics_to_rx.append(topic)

        else:
            raise KeyError('Key %s is not in schoolbus tester' % item)
        return new_val

    def subscribeTopicOrPattern(self, topic_or_pattern):
        # If the topic contains an asterisk, then
        # build a regex pattern, rather than passing
        # the topic itself:
        try:
            topic_or_pattern.index('*')
            # Topic is a pattern of topics:
            topic = re.compile(topic_or_pattern)
        except ValueError:
            # Not a pattern, just a regular topic name:
            topic = topic_or_pattern
        self.testBus.subscribeToTopic(topic,
                                      functools.partial(self.messageReceiver))

    def unsubscribeFromTopicOrPattern(self, topic_or_pattern):
        # If the topic contains an asterisk, then
        # build a regex pattern, rather than passing
        # the topic itself:
        try:
            topic_or_pattern.index('*')
            # Topic is a pattern of topics:
            topic = re.compile(topic_or_pattern)
        except ValueError:
            # Not a pattern, just a regular topic name:
            topic = topic_or_pattern
        self.testBus.unsubscribeFromTopic(topic)

    def createMessage(self, topic=None, msgLen=None, content=None):
        '''
        Returns a BusMessage whose content is the given length,
        and whose topicName is the given topic.
        
        :param topic: topic for which this message is destined.
        :type topic: string
        :param msgLen: desired str length
        :type msgLen: int
        :param content: content of the message. If None, a random content
            of length msgLen will be created
        :return: BusMessage object ready to publish. The context field of the
            message instance will be the MD5 of the content.
        :rtype: BusMessage
        '''

        if topic is None:
            topic = self._stream_topic

        if msgLen is None:
            msgLen = self.standard_msg_len

        if content is None:
            content = self.createRandomStr(msgLen)

        msg = BusMessage(content=content,
                         topicName=topic,
                         context=hashlib.md5(str(content)).hexdigest())

        return msg

    def createRandomStr(self, the_len):
        the_len = int(the_len)
        content = bytearray()
        for _ in range(the_len):
            content.append(random.choice(string.letters))
        return str(content)

    def sendMessage(self, bus_msg=None):
        '''
        Publish the given BusMessage, or
        the standard message:
        '''
        if bus_msg is None:
            bus_msg = self.standard_oneshot_msg
        self.testBus.publish(bus_msg)

    def logInfo(self, msg):
        if self.loglevel >= OnDemandPublisher.LOG_LEVEL_INFO:
            print(str(datetime.datetime.now()) + ' info: ' + msg)

    def logErr(self, msg):
        if self.loglevel >= OnDemandPublisher.LOG_LEVEL_ERR:
            print(str(datetime.datetime.now()) + ' error: ' + msg)

    def logDebug(self, msg):
        if self.loglevel >= OnDemandPublisher.LOG_LEVEL_DEBUG:
            print(str(datetime.datetime.now()) + ' debug: ' + msg)

    def syntaxCheckReceiver(self, busMsg, context=None):
        '''
        Given a BusMessage instance, determine whether it contains 
        all necessary attributes: ID, and time. This method 
        is a callback for topic specified in module variable
        self.syntaxTopic. Constructs a string that lists any
        errors or warnings, and returns that string as a
        synchronous response.
        
        :param busMsg: the message to be evaluated
        :type busMsg: BusMessage
        :param context: not used
        :type context: <any>
        '''
        errors = []

        if busMsg.id is None:
            errors.append('Error: No ID field')
        if type(busMsg.time) != int and type(busMsg.time) != float:
            errors.append("Error: Time must be an int, was '%s'" % busMsg.time)
            try:
                busMsg.isoTime
            except (ValueError, TypeError):
                errors.append(
                    "Error: Time value not transformable to ISO time '%s'" %
                    busMsg.time)
        try:
            json.loads(busMsg.content)
        except AttributeError:
            errors.append("Warning: no 'content' field.")
        except ValueError:
            errors.append(
                "Warning: 'content' field present, but not valid JSON.")

        if len(errors) == 0:
            response = 'Message OK'
        else:
            response = self.testBus.makeResponseMsg(busMsg, '; '.join(errors))

        self.testBus.publish(response)

    def messageReceiver(self, busMsg, context=None):
        '''
        Method that is called with each received message.
        
        :param busMsg: bus message object
        :type busMsg: BusMessage
        :param context: context Python structure, if subscribeToTopic() was
            called with one.uu
        :type context: <any>
        '''

        self.mostRecentRxTime = time.time()
        self.printedResetting = False

        if self.serve_echo and busMsg.topicName == 'echo':
            # Create a message with the same content as the incoming
            # message, timestamp the new message, and publish it
            # on the response topic (i.e. tmp.<msgId>):
            respMsg = self.testBus.makeResponseMsg(busMsg, busMsg.content)

            # Publish a response:
            self.testBus.publish(respMsg)
            self.numEchoed += 1

            if self.numEchoed % 1000 == 0:
                print('Echoed %d' % self.numEchoed)

            if self.numEchoed % 10000 == 0:
                # Messages per second:
                msgs_per_sec = 10000.0 / (time.time() - self.batch_start_time)
                print('Echoing: %f Msgs/sec' % msgs_per_sec)
                self.batch_start_time = time.time()

        self.numReceived += 1
        if self.numReceived % 1000 == 0:
            stats_msg = 'Rxed %d' % self.numReceived
            self.logInfo(stats_msg)
            self.bus_stats_queue.put_nowait(stats_msg)

        if self.numReceived % 10000 == 0:
            # Messages per second:
            msgs_per_sec = 10000.0 / (time.time() - self.batch_start_time)
            stats_msg1 = 'Rx (no echo): %f Msgs/sec' % msgs_per_sec
            self.logInfo(stats_msg1)
            if not self.bus_stats_queue.full():
                self.bus_stats_queue.put_nowait(stats_msg1)
            self.batch_start_time = time.time()

        topic = busMsg.topicName
        matching_subscribed_topic = self.findTopic(topic, self.topics_to_rx)
        if matching_subscribed_topic is not None:
            if not self.bus_msg_queue.full():
                self.bus_msg_queue.put_nowait(str(datetime.datetime.now()) +\
                                          "--%s: " % matching_subscribed_topic +\
                                          busMsg.content)

    def findTopic(self, topicIdentifier, topicStrArr):
        '''
        Given either a string or a pattern, and an
        array of strings that correspond to topic names
        return return the topic name that matches the
        topicIdentifier, or None. The strings in topicStrArr
        may be regular expressions as used in pattern subscriptions.
        In that case a regex match is done against the topicIdentifier.
        
        :param topicIdentifier: a string that names a topic. This is
            a straight name, not a regex
        :type topicIdentifier: string
        :param topicsStrArr: array of strings, some of which may contain
            the wildcard char '*', indicating that the string is a 
            regex pattern. In such a case the topicIdentifier is
            checked against that regex pattern.
        :return: the string in topicsStrArr that matched, or None. 
            The returned string may, of course be a regex string if
            that's what caused a match.
        :rType: {string | None}
        '''

        for topic_in_list in topicStrArr:
            # Is it a pattern?
            asterisk_pos = string.find(topic_in_list, '*')
            if asterisk_pos == -1:
                # Simple topic name, check exact match:
                if topicIdentifier == topic_in_list:
                    return topic_in_list
            else:
                # Topic 'name' in the list is actually a pattern:
                if re.match(topic_in_list, topicIdentifier) is not None:
                    return topic_in_list

        # No match:
        return None

    def resetEchoedCounter(self):
        currTime = time.time()

        if self.mostRecentRxTime is None:
            # Nothing received yet:
            self.startTime = time.time()
            self.batch_start_time = time.time()
            self.startIdleTimer()
            return

        if currTime - self.mostRecentRxTime <= OnDemandPublisher.MAX_IDLE_TIME:
            # Received msgs during more recently than idle time:
            self.startIdleTimer()
            return

        # Did not receive msgs within idle time:
        self.printTiming()
        if not self.printedResetting:
            resetMsg = 'Resetting (echoed %d)' % self.numEchoed
            self.logInfo(resetMsg)
            self.bus_stats_queue.put_nowait(resetMsg)
            self.printedResetting = True
        self.numEchoed = 0
        self.startTime = time.time()
        self.batch_start_time = time.time()
        self.timer = self.startIdleTimer()

    def startIdleTimer(self):
        threading.Timer(OnDemandPublisher.MAX_IDLE_TIME,
                        functools.partial(self.resetEchoedCounter)).start()

    def stop(self, signum=None, frame=None):
        try:
            self.logDebug('Unsubscribing from echo...')
            self.serve_echo = False
            self.logDebug('Unsubscribing from check_syntax...')
            self.check_syntax = False

            self.logInfo('Shutting down streamer thread...')
            self.msg_streamer.stop(signum=None, frame=None)
            self.msg_streamer.join(JOIN_WAIT_TIME)
            if self.msg_streamer.is_alive():
                raise TimeoutError(
                    "Unable to stop message streamer thread '%s'." %
                    self.msg_streamer.name)

        except Exception as e:
            print("Error while shutting down message streamer thread: '%s'" %
                  ` e `)

        self.done = True
        self.interruptEvent.set()

    def printTiming(self, startTime=None):
        currTime = time.time()
        if startTime is None:
            startTime = self.startTime
            return
        echoMsg = 'Echoed %d messages' % self.numEchoed
        self.logInfo(echoMsg)
        if self.numEchoed > 0:
            timeElapsed = float(currTime) - float(self.startTime)
            rateMsg = 'Msgs per second: %d' % (timeElapsed / self.numEchoed)
            self.logInfo(rateMsg)
            self.bus_stats_queue.put_nowait(rateMsg)

    def run(self):
        while not self.done:
            self.startTime = time.time()
            self.startIdleTimer()
            self.interruptEvent.wait()
            if not self.done and self.sendMsg:
                self.testBus.publish(self.outMsg)
                self.interruptEvent.clear()
                self.sendMsg = False
            continue

        self.testBus.unsubscribeFromTopic(self._echo_topic)
        self.testBus.close()
    def __init__(self, serveEchos=True, 
                 listenOn=None, 
                 streamMsgs=False, 
                 checkSyntax=True,
                 oneShotMsg=None):
        '''
        Initialize the test harness server. 
        
        :param serveEchos: if True receives messages on ECHO_TOPIC, and returns  
            message with the same content, but different timestamp as a synchronous call. 
        :type serveEchos: bool
        :param listenOn: if this parameter provides an array of topic(s), messages on these
            topics are received and counted, then placed on self.bus_msg_queue. Intermittent reception counts
            and reception rates are printed to the console, and place on self.bus_stats_queue.
        :type listenOn: {None | [string]}
        :param streamMsgs: if True, a continuous stream of messages are sent to
            STREAM_TOPIC. The timestamp is changed each time.
        :type streamMsgs: bool
        :param checkSyntax: if true, listens on SYNTAX_TOPIC, checks incoming messages
            for conformance to SchoolBus message format, and synchronously returns a result.
        :type checkSyntax: bool
        :param oneShotMsg: if non-none this parameter is expected to be a two-tuple
            containing a topic and a content. Either may be None. If topic is None,
            the message is sent to STREAM_TOPIC. If content is None, content will be
            a random string of length STANDARD_MSG_LENGTH.
        :type oneShotMsg: { None | ({string | None}, {string | None})}
        '''
        threading.Thread.__init__(self)
        
        self.loglevel = OnDemandPublisher.LOG_LEVEL_DEBUG
        #self.loglevel = OnDemandPublisher.LOG_LEVEL_INFO
        #self.loglevel = OnDemandPublisher.LOG_LEVEL_NONE
        
        #***********
#         print ('serveEchos %s' % serveEchos)
#         print ('listenOn %s' % listenOn)
#         print ('streamMsgs %s' % streamMsgs)
#         print('checkSyntax %s' % checkSyntax)
#         print('oneShotMsg %s' % str(oneShotMsg))
#         sys.exit()
        #***********
        
        self._serveEchos = serveEchos
        
        self.daemon = True
        self.testBus = BusAdapter()
        self.done = False
        # No topics yet that we are to listen to, but drop msgs for:
        self.topics_to_rx = []
        
        self.echo_topic = ECHO_TOPIC
        self.syntax_check_topic = SYNTAX_TOPIC
        self.standard_msg_len = STANDARD_MSG_LENGTH
        self._stream_interval = STREAM_INTERVAL
        
        self._stream_topic = STREAM_TOPIC
        self.standard_bus_msg = self.createMessage(STREAM_TOPIC, STANDARD_MSG_LENGTH, content=None)
        
        self.one_shot_topic = STREAM_TOPIC
        self.one_shot_content = self.standard_bus_msg
        
        # Queue into which incoming messages are fed
        # for consumers to communicated to clients, such
        # as a Web UI:
        self.bus_msg_queue   = Queue.Queue(MAX_SAVED_MSGS)
        
        # Queue to which aggregate stats will be written:  
        self.bus_stats_queue = Queue.Queue(MAX_SAVED_MSGS) 

        # Create a 'standard' message to send out
        # when asked to via SIGUSR1 or the Web:

        if oneShotMsg is not None:
            # The oneShotMsg arg may have one of the following values: 
            #   - None if oneshot was not requested. 
            #   - An empty array if -o (or --oneshot) 
            #        was listed in the command line, 
            #        but neither topic nor message content
            #        were specified.
            #   - A one-element array if the topic was
            #        specified
            #   - A two-element array if both topic and
            #        content were specified.

            (one_shot_topic, self.one_shot_content) = oneShotMsg
            if one_shot_topic is not None:
                self.one_shot_topic = one_shot_topic
            
            # Create a message that one can have sent on demand
            # by sending a SIGUSR1 to the server, or using
            # the Web UI:

            self.standard_oneshot_msg = self.createMessage(self.one_shot_topic, content=self.one_shot_content)
            # Send the standard msg:
            self.sendMessage(self.standard_oneshot_msg)
        else:
            # Create a standard message with default topic/content:
            self.standard_oneshot_msg = self.createMessage()
       
        # Start time of a 10,000 messages batch:
        self.batch_start_time = time.time()        
        
        # Subscribe, and ensure that context is delivered
        # with each message:

        if serveEchos:
            self.serve_echo = True
        
        if listenOn is not None:
            # Array of topics to listen to 
            # If we are to listen to some (additional) topic(s),
            # on which no action is taken, subscribe to it/them now:
            self['topicsToRx'] = listenOn

        if checkSyntax:
            self.check_syntax = True
            
        self.interruptEvent = threading.Event()
        
        # Number of messages received and echoed:
        self.numEchoed = 0
        # Alternative to above when not echoing: Number of msgs just received:
        self.numReceived = 0
        self.mostRecentRxTime = None
        self.printedResetting = False
        
        # Not asked to send a message yet.
        self.sendMsg = False
        
        if streamMsgs is not None:
            # The streamMsgs arg may have one of the following values: 
            #   - None if streaming messages was not requested. 
            #   - An empty array if -s (or --streamMsgs) 
            #        was listed in the command line, 
            #        but neither topic nor message content
            #        were specified.
            #   - A one-element array if the topic was
            #        specified
            #   - A two-element array if both topic and
            #        content were specified.

            self.msg_streamer = MessageOutStreamer(self.standard_bus_msg)
            
            # The caller passed in a tuple with
            # topic and content for streaming. If they
            # are non-None, update the streamer:
            
            (stream_topic, stream_content) = streamMsgs
            if stream_topic is not None:
                self.stream_topic = stream_topic
            if stream_content is not None:
                self.stream_content = stream_content

            # Start streaming:
            
            self.msg_streamer.pause(True)
            self.msg_streamer.start()
            
        else:
            # Create a standard message with default topic/content:
            self.standard_stream_msg = self.createMessage()
            self.msg_streamer = None
Exemplo n.º 37
0
class RedisPerformanceTester(object):
    '''
    classdocs
    '''

    PUBLISH_TOPIC = 'test'

    def __init__(self):
        '''
        Constructor
        '''
        self.host = 'localhost'
        self.port = 6379

        self.letterChoice = string.letters
        self.bus = BusAdapter()

    def close(self):
        self.bus.close()

    def publishToUnsubscribedTopic(self, numMsgs, msgLen, block=True):
        '''
        Publish given number of messages to a topic that
        nobody is subscribed to. Each msg is of length msgLen.
        
        :param numMsgs:
        :type numMsgs:
        :param msgLen:
        :type msgLen:
        '''
        (msg, md5) = self.createMessage(msgLen)  #@UnusedVariable
        busMsg = BusMessage(content=msg,
                            topicName=RedisPerformanceTester.PUBLISH_TOPIC)
        startTime = time.time()
        for _ in range(numMsgs):
            self.bus.publish(busMsg, block=block)
        endTime = time.time()
        self.printResult(
            'Publishing %s msgs to empty space (block==%s): ' %
            (str(numMsgs), str(block)), startTime, endTime, numMsgs)

    def publishToSubscribedTopic(self,
                                 numMsgs,
                                 msgLen,
                                 block=True,
                                 sameProcessListener=True):
        '''
        Publish given number of messages to a topic that
        a thread in this same process is subscribed to.
        These are asynchronous publish() calls. The receiving
        thread computes MD5 of the received msg to verify.
        
        :param numMsgs:
        :type numMsgs:
        :param msgLen:
        :type msgLen:
        :param block: if True, publish() will wait for server confirmation.
        :type block: bool
        :param sameProcessListener: if True, the listener to messages will be a thread in this
            Python process (see below). Else an outside process is expected to be subscribed
            to RedisPerformanceTester.PUBLISH_TOPIC.
        :type sameProcessListener: bool
        '''

        (msg, md5) = self.createMessage(msgLen)
        busMsg = BusMessage(content=msg,
                            topicName=RedisPerformanceTester.PUBLISH_TOPIC)
        try:
            listenerThread = ReceptionTester(msgMd5=md5, beSynchronous=False)
            listenerThread.daemon = True
            listenerThread.start()

            startTime = time.time()
            for _ in range(numMsgs):
                self.bus.publish(busMsg, block=block)
            endTime = time.time()
            self.printResult(
                'Publishing %s msgs to a subscribed topic (block==%s): ' %
                (str(numMsgs), str(block)), startTime, endTime, numMsgs)
        except Exception:
            raise
        finally:
            listenerThread.stop()
            listenerThread.join(3)

    def syncPublishing(self, numMsgs, msgLen, block=True):

        #sys.stdout.write('Run python src/redis_bus_python/test/test_harness_server.py echo and hit ENTER...')
        #sys.stdin.readline()

        (msg, md5) = self.createMessage(msgLen)  #@UnusedVariable

        busMsg = BusMessage(content=msg, topicName=ECHO_TOPIC)

        startTime = time.time()
        try:
            for serialNum in range(numMsgs):
                try:
                    busMsg.id = serialNum
                    res = self.bus.publish(busMsg,
                                           sync=True,
                                           timeout=5,
                                           block=block)  #@UnusedVariable
                except SyncCallTimedOut:
                    #printThreadTraces()
                    raise

            endTime = time.time()
            self.printResult(
                'Publishing %s synch msgs (block==%s): ' %
                (str(numMsgs), str(block)), startTime, endTime, numMsgs)
        finally:
            pass

    def rawIronPublish(self, numMsgs, msgLen, block=True):

        sock = self.bus.topicWaiterThread.pubsub.connection._sock
        sock.setblocking(1)
        sock.settimeout(2)

        (msg, md5) = self.createMessage(msgLen)  #@UnusedVariable
        wireMsg = '*3\r\n$7\r\nPUBLISH\r\n$4\r\ntest\r\n$190\r\n{"content": \r\n "dRkLSUQxFVSHuVnEekLtfPsXULtWEESQwaRYZtxpFGYRGphNTkRQAMPJfDxoGKOKPCMmptZBriVVfV\r\n LvYisehirsYSHdDrhXRgGl", "id": "a62b8cde-6bf6-4f75-a1f3-e768bec4d5e1", "time": 1437516321946}\r\n'
        num_sent = 0
        start_time = time.time()
        try:
            for _ in range(numMsgs):
                sock.sendall(wireMsg)
                num_sent += 1
                #             if num_sent % 1000 == 0:
                #                 print('Sent %d' % num_sent)
                if block:
                    #time.sleep(0.01)
                    numListeners = sock.recv(1024)  #@UnusedVariable
        except socket.timeout:
            end_time = time.time()
            self.printResult(
                'Sending on raw socket; result timeout after %d msgs.' %
                num_sent, start_time, end_time, numMsgs)
            sys.exit()
        except Exception:
            end_time = time.time()
            self.printResult(
                'Sending on raw socket; error %d msgs.' % num_sent, start_time,
                end_time, numMsgs)
            raise

        end_time = time.time()
        self.printResult(
            'Sent %d msgs on raw socket; block==%s' % (numMsgs, block),
            start_time, end_time, numMsgs)

    def justListenAndCount(self):
        listenerThread = ReceptionTester(beSynchronous=False)
        listenerThread.daemon = True
        listenerThread.start()
        signal.pause()
        listenerThread.stop()
        listenerThread.join()

    def createMessage(self, msgLen):
        '''
        Returns a string of a given length, and its checksum
        
        :param msgLen: desired str length
        :type msgLen: int
        '''

        msg = bytearray()
        for _ in range(msgLen):
            msg.append(random.choice(self.letterChoice))
        return (str(msg), hashlib.md5(str(msg)).hexdigest())

    def _connect(self):

        err = None
        # Get addr options for Redis host/port, with arbitrary
        # socket family (the 0), and stream type:
        for res in socket.getaddrinfo(self.host, self.port, 0,
                                      socket.SOCK_STREAM):
            family, socktype, proto, canonname, socket_address = res  #@UnusedVariable
            sock = None
            try:
                sock = socket.socket(family, socktype, proto)
                # TCP_NODELAY
                sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)

                sock.connect((self.host, self.port))
                return sock

            except socket.error as _:
                err = _
                if sock is not None:
                    sock.close()

        if err is not None:
            raise err
        raise socket.error("socket.getaddrinfo returned an empty list")

    def printResult(self, headerMsg, startTime, endTime, numMsgs):
        totalTime = endTime - startTime
        timePerMsg = totalTime / numMsgs
        msgPerSec = numMsgs / totalTime
        print(headerMsg)
        print('msgsPerSec: %s' % str(msgPerSec))
        print('timePerMsg: %s' % str(timePerMsg))
class MessageOutStreamer(threading.Thread):
    '''
    Thread that keeps sending the same message over and over,
    except for changing the time field.
    '''
    
    def __init__(self, busMsg=None):
        super(MessageOutStreamer, self).__init__()
        
        self.daemon = True
        self.streamBus = BusAdapter()
        self.busMsg = BusMessage() if busMsg is None else busMsg
    
        self.done = False
        self._paused = False
        self._stream_interval = STREAM_INTERVAL
        
    def pause(self, do_pause):
        if do_pause:
            self._paused = True
        else:
            self._paused = False

    @property
    def paused(self):
        return self._paused
    
    @property
    def stream_interval(self):
        return self._stream_interval
    
    @stream_interval.setter
    def stream_interval(self, new_val):
        '''
        Note: we trust that caller ensures new_val to 
        be a non-negative float. 
        
        :param new_val: number of (fractional) seconds to wait between
            stream messages. Zero means no wait.
        :type new_val:float
        '''
        self._stream_interval = new_val
    
    def change_stream_topic(self, newTopic):
        self.busMsg.topicName = newTopic
    
    def change_stream_content(self, newContent):
        self.busMsg.content = newContent

    def run(self):
        '''
        Keep publishing one message over and over,
        
        :param msgLen: length of payload
        :type msgLen: int
        '''
        while not self.done:
            self.busMsg.time = time.time()
            self.streamBus.publish(self.busMsg)
            while self._paused and not self.done:
                time.sleep(1)
            if self._stream_interval > 0:
                time.sleep(self._stream_interval)
            
    def stop(self, signum, frame):
        self.done = True
        self.streamBus.close()
class OnDemandPublisher(threading.Thread):
    '''
    Server for testing Redis bus modules. Started from the
    command line, or imported into applications. Serves multiple
    functions, individually or together:
    
        1. echoes messages it receives on topic ECHO_TOPIC.
        2. listens to messages without doing anything
        3. sends a continuous stream of messages
        4. check the syntax of an incoming msg, returning the result
        5. send one message on demand.

	USAGE: test_harness_server.py [-h] [-e] [-s [topic [content ...]]] [-c]
	                              [-l topic [topic ...]]
	                              [-o [topic [content ...]]]
	
	optional arguments:
	  -h, --help            show this help message and exit
	  -e, --echo            Echo messages arriving on topic 'echo' as sychronous replies.
	  -s [topic [content ...]], --streamMsgs [topic [content ...]]
	                        Send the same bus message over and over. Topic, or both,
	                        topic and content may be provided. If content is omitted,
	                        a random string of 100 characters will be used.
	                        If topic is omitted as well, messages will be streamed to 'test'
	  -c, --checkSyntax     Check syntax of messages arriving on topic 'bus_syntax'; 
	                        synchronously return result report.
	  -l topic [topic ...], --listenOn topic [topic ...]
	                        Subscribe to given topic(s), and throw the messages away
	  -o [topic [content ...]], --oneshot [topic [content ...]]
	                        Given a topic and a content string, send a bus message. Topic, or both 
	                        topic and content may be provided. If content is omitted,
	                        a random string of 100 characters will be used.
	                        If topic is omitted as well, messages will be streamed to 'test'
	    

    The thread always listens on the ECHO_TOPIC. But
    received messages are only **echoed** if the service was
    started with the 'echo' option
    
    The echo function acts like a synch-call service
    should: echoing on the response topic, which it
    derives from the msg ID. The content will be the
    content of the incoming message, the timestamp
    will be the receipt time.
    
    When echoing, the service keeps track of how many messages 
    it has echoed. But after not receiving any messages for 
    OnDemandPublisher.MAX_IDLE_TIME seconds, the
    current count is printed, and the counter is 
    reset to zero. After every 1000 messages, the total
    number echoed is printed to the console. Every 10,000
    messages, the message/sec rate is printed as well.
    
    When asked to **listen** to some topic, messages on that
    topic are received and counted. Then the message is
    placed into a queue where it can be picked up (self.bus_msg_queue). 
    If nobody attends to the queue, it fills up, and then sits there;
    i.e. no big harm. Statistics printing is as for echoing.
    Stats are placed on the bus_stats_queue for consumption by anyone
    interested (or not).
    
    Messages sent in a **stream** by sendMessageStream()
    contain their content's MD5 in the context field. This constant
    message sending is initiated only if the service is
    started with the 'send_stream' option.

    When asked to check message **syntax**, the service listens
    on SYNTAX_TOPIC, and synchronously returns a string 
    reporting on violations of SchoolBus specifications in 
    the incoming message.
    
    A **oneshot message** is sent if the server is started
    with the --oneshot option. If topic and/or content are provided,
    the message is loaded with the given content and published to
    the given topic. Defaults are to fill the content with a random
    string, and to publish to STREAM_TOPIC.
    
    '''
    
    # ----------------------------  Constants ----------------

    # Maximum time for no message to arrive before
    # starting over counting messages:
    MAX_IDLE_TIME = 5
    
    FIND_COMMA_PATTERN = re.compile(r'[,]+')
    FIND_SPACE_PATTERN = re.compile(r'[ ]+')
    
    LOG_LEVEL_NONE  = 0
    LOG_LEVEL_ERR   = 1
    LOG_LEVEL_INFO  = 2
    LOG_LEVEL_DEBUG = 3
    
    def __init__(self, serveEchos=True, 
                 listenOn=None, 
                 streamMsgs=False, 
                 checkSyntax=True,
                 oneShotMsg=None):
        '''
        Initialize the test harness server. 
        
        :param serveEchos: if True receives messages on ECHO_TOPIC, and returns  
            message with the same content, but different timestamp as a synchronous call. 
        :type serveEchos: bool
        :param listenOn: if this parameter provides an array of topic(s), messages on these
            topics are received and counted, then placed on self.bus_msg_queue. Intermittent reception counts
            and reception rates are printed to the console, and place on self.bus_stats_queue.
        :type listenOn: {None | [string]}
        :param streamMsgs: if True, a continuous stream of messages are sent to
            STREAM_TOPIC. The timestamp is changed each time.
        :type streamMsgs: bool
        :param checkSyntax: if true, listens on SYNTAX_TOPIC, checks incoming messages
            for conformance to SchoolBus message format, and synchronously returns a result.
        :type checkSyntax: bool
        :param oneShotMsg: if non-none this parameter is expected to be a two-tuple
            containing a topic and a content. Either may be None. If topic is None,
            the message is sent to STREAM_TOPIC. If content is None, content will be
            a random string of length STANDARD_MSG_LENGTH.
        :type oneShotMsg: { None | ({string | None}, {string | None})}
        '''
        threading.Thread.__init__(self)
        
        self.loglevel = OnDemandPublisher.LOG_LEVEL_DEBUG
        #self.loglevel = OnDemandPublisher.LOG_LEVEL_INFO
        #self.loglevel = OnDemandPublisher.LOG_LEVEL_NONE
        
        #***********
#         print ('serveEchos %s' % serveEchos)
#         print ('listenOn %s' % listenOn)
#         print ('streamMsgs %s' % streamMsgs)
#         print('checkSyntax %s' % checkSyntax)
#         print('oneShotMsg %s' % str(oneShotMsg))
#         sys.exit()
        #***********
        
        self._serveEchos = serveEchos
        
        self.daemon = True
        self.testBus = BusAdapter()
        self.done = False
        # No topics yet that we are to listen to, but drop msgs for:
        self.topics_to_rx = []
        
        self.echo_topic = ECHO_TOPIC
        self.syntax_check_topic = SYNTAX_TOPIC
        self.standard_msg_len = STANDARD_MSG_LENGTH
        self._stream_interval = STREAM_INTERVAL
        
        self._stream_topic = STREAM_TOPIC
        self.standard_bus_msg = self.createMessage(STREAM_TOPIC, STANDARD_MSG_LENGTH, content=None)
        
        self.one_shot_topic = STREAM_TOPIC
        self.one_shot_content = self.standard_bus_msg
        
        # Queue into which incoming messages are fed
        # for consumers to communicated to clients, such
        # as a Web UI:
        self.bus_msg_queue   = Queue.Queue(MAX_SAVED_MSGS)
        
        # Queue to which aggregate stats will be written:  
        self.bus_stats_queue = Queue.Queue(MAX_SAVED_MSGS) 

        # Create a 'standard' message to send out
        # when asked to via SIGUSR1 or the Web:

        if oneShotMsg is not None:
            # The oneShotMsg arg may have one of the following values: 
            #   - None if oneshot was not requested. 
            #   - An empty array if -o (or --oneshot) 
            #        was listed in the command line, 
            #        but neither topic nor message content
            #        were specified.
            #   - A one-element array if the topic was
            #        specified
            #   - A two-element array if both topic and
            #        content were specified.

            (one_shot_topic, self.one_shot_content) = oneShotMsg
            if one_shot_topic is not None:
                self.one_shot_topic = one_shot_topic
            
            # Create a message that one can have sent on demand
            # by sending a SIGUSR1 to the server, or using
            # the Web UI:

            self.standard_oneshot_msg = self.createMessage(self.one_shot_topic, content=self.one_shot_content)
            # Send the standard msg:
            self.sendMessage(self.standard_oneshot_msg)
        else:
            # Create a standard message with default topic/content:
            self.standard_oneshot_msg = self.createMessage()
       
        # Start time of a 10,000 messages batch:
        self.batch_start_time = time.time()        
        
        # Subscribe, and ensure that context is delivered
        # with each message:

        if serveEchos:
            self.serve_echo = True
        
        if listenOn is not None:
            # Array of topics to listen to 
            # If we are to listen to some (additional) topic(s),
            # on which no action is taken, subscribe to it/them now:
            self['topicsToRx'] = listenOn

        if checkSyntax:
            self.check_syntax = True
            
        self.interruptEvent = threading.Event()
        
        # Number of messages received and echoed:
        self.numEchoed = 0
        # Alternative to above when not echoing: Number of msgs just received:
        self.numReceived = 0
        self.mostRecentRxTime = None
        self.printedResetting = False
        
        # Not asked to send a message yet.
        self.sendMsg = False
        
        if streamMsgs is not None:
            # The streamMsgs arg may have one of the following values: 
            #   - None if streaming messages was not requested. 
            #   - An empty array if -s (or --streamMsgs) 
            #        was listed in the command line, 
            #        but neither topic nor message content
            #        were specified.
            #   - A one-element array if the topic was
            #        specified
            #   - A two-element array if both topic and
            #        content were specified.

            self.msg_streamer = MessageOutStreamer(self.standard_bus_msg)
            
            # The caller passed in a tuple with
            # topic and content for streaming. If they
            # are non-None, update the streamer:
            
            (stream_topic, stream_content) = streamMsgs
            if stream_topic is not None:
                self.stream_topic = stream_topic
            if stream_content is not None:
                self.stream_content = stream_content

            # Start streaming:
            
            self.msg_streamer.pause(True)
            self.msg_streamer.start()
            
        else:
            # Create a standard message with default topic/content:
            self.standard_stream_msg = self.createMessage()
            self.msg_streamer = None

    @property
    def running(self):
        return not self.done

    @property
    def echo_topic(self):
        return self._echo_topic
    
    @echo_topic.setter
    def echo_topic(self, new_echo_topic):
        if new_echo_topic is None or len(new_echo_topic) == 0:
            # Set to default echo topic:
            new_echo_topic = ECHO_TOPIC 
        self._echo_topic = new_echo_topic

    @property
    def serve_echo(self):
        return self._serveEchos

    @serve_echo.setter
    def serve_echo(self, do_serve):
        if do_serve:
            # Have incoming messages delivered to messageReceiver() not
            # via a queue, but by direct call from the library:
            self.testBus.subscribeToTopic(self.echo_topic, functools.partial(self.messageReceiver), threaded=False)
            self._serveEchos = True
        else:
            self.testBus.unsubscribeFromTopic(self.echo_topic)
            self._serveEchos = False

    @property
    def stream_topic(self):
        return self.msg_streamer.busMsg.topicName
    
    @stream_topic.setter
    def stream_topic(self, new_stream_topic):
        if new_stream_topic is None or len(new_stream_topic) == 0:
            # Set to default stream topic:
            new_stream_topic = STREAM_TOPIC 
        self.msg_streamer.change_stream_topic(new_stream_topic)

    @property
    def stream_content(self):
        return self.msg_streamer.busMsg.content
    
    @stream_content.setter
    def stream_content(self, new_stream_content):
        if new_stream_content is None or len(new_stream_content) == 0:
            self.msg_streamer.change_stream_content(self.createRandomStr(self.standard_msg_len))
        else:
            self.msg_streamer.change_stream_content(new_stream_content)

    @property
    def streaming(self):
        return not self.msg_streamer.paused
    
    @streaming.setter
    def streaming(self, should_stream):
        should_pause = not should_stream
        self.msg_streamer.pause(should_pause)

    @property
    def stream_interval(self):
        return self._stream_interval
    
    @stream_interval.setter
    def stream_interval(self, new_interval):
        # If the new value is not a number or empty str, set to default:
        if type(new_interval) == str  and len(new_interval) == 0:
            new_interval = STREAM_INTERVAL
        else:
            # Non-empty string or number; 
            # Ensure float, and replace negative
            # values with 0:
            try:
                if float(new_interval) < 0:
                    new_interval = 0.0
            except ValueError:
                raise ValueError("Attempt to set streaming interval to a non-numeric quantity: '%s'" % str(new_interval))
        
        self.msg_streamer.stream_interval = new_interval
        self._stream_interval = new_interval


    @property
    def one_shot_topic(self):
        return self._one_shot_topic
    
    @one_shot_topic.setter
    def one_shot_topic(self, new_one_shot_topic):
        if new_one_shot_topic is None or len(new_one_shot_topic) == 0:
            # Set to default oneshot topic:
            new_one_shot_topic = STREAM_TOPIC
        else:
            self._one_shot_topic = new_one_shot_topic
        
    @property
    def one_shot_content(self):
        return self._one_shot_content
    
    @one_shot_content.setter
    def one_shot_content(self, new_one_shot_content):
        if new_one_shot_content is None or len(new_one_shot_content) == 0:
            self._one_shot_content = self.createRandomStr(self.standard_msg_len)
        else:
            self._one_shot_content = new_one_shot_content


    @property
    def syntax_check_topic(self):
        return self._syntax_check_topic
    
    @syntax_check_topic.setter
    def syntax_check_topic(self, new_syntax_check_topic):
        if new_syntax_check_topic is None or len(new_syntax_check_topic) == 0:
            # Set to default syntax check topic:
            new_syntax_check_topic = SYNTAX_TOPIC
        self._syntax_check_topic = new_syntax_check_topic

    @property
    def check_syntax(self):
        return self._checkSyntax

    @check_syntax.setter
    def check_syntax(self, do_check):
        if do_check:
            self.testBus.subscribeToTopic(self.syntax_check_topic, functools.partial(self.syntaxCheckReceiver))
            self._checkSyntax = True
        else:
            self.testBus.unsubscribeFromTopic(self.syntax_check_topic)
            self._checkSyntax = False

    @property
    def standard_msg_len(self):
        return self._standard_msg_len
    
    @standard_msg_len.setter
    def standard_msg_len(self, new_standard_msg_len):
        if new_standard_msg_len == 0 or \
           new_standard_msg_len is None:
            new_standard_msg_len = STANDARD_MSG_LENGTH
        try:
            new_standard_msg_len = int(new_standard_msg_len)
        except:
            new_standard_msg_len = STANDARD_MSG_LENGTH
            
        self._standard_msg_len = new_standard_msg_len
    
    def __getitem__(self, item):
        if item == 'echoTopic':
            return self.echo_topic

        elif item == 'echo':
            return self.testBus.subscribedTo(self['echoTopic'])
        
        elif item == 'streamTopic':
            return self.stream_topic
        elif item == 'streamContent':
            return self.stream_content
        elif item == 'streaming':
            return self.streaming
        elif item == 'streamInterval':
            return self.stream_interval
        
        
        elif item == 'oneShotTopic':
            return self.one_shot_topic
        elif item == 'oneShotContent':
            if isinstance(self.one_shot_content, BusMessage):
                return self.one_shot_content.content
            else:
                return self.one_shot_content
        
        elif item == 'strLen':
            return self.standard_msg_len
        
        elif item == 'syntaxTopic':
            return self.syntax_check_topic
        
        elif item == 'chkSyntax':
            return self.testBus.subscribedTo(self['syntaxTopic'])

        
        elif item == 'topicsToRx':
            return self.topics_to_rx
        
        else:
            raise KeyError('Key %s is not in schoolbus tester' % item)
    
    def __setitem__(self, item, new_val):
                
        if item == 'echoTopic':
            self.echo_topic = new_val
            
        elif item == 'echo':
            # Deal with both string and bool:
            new_val = (new_val == 'True' or new_val == True)
            self.serve_echo = new_val

        elif item == 'streamTopic':
            self.stream_topic = new_val
        elif item == 'streamContent':
            self.stream_content = new_val
        elif item == 'streaming':
            # Deal with both string and bool:
            new_val = (new_val == 'True' or new_val == True)
            self.streaming = new_val
        elif item == 'streamInterval':
            self.stream_interval = new_val
            
            
        elif item == 'oneShotContent':
            self.one_shot_content = new_val
        
        elif item == 'oneShotTopic':
            self.one_shot_topic = new_val
        elif item == 'oneShotContent':
            self.one_shot_content = new_val
            
        elif item == 'strLen':
            self.standard_msg_len = new_val
            
        elif item == 'syntaxTopic':
            self.syntax_check_topic = new_val
            
        elif item == 'chkSyntax':
            # Deal with both string and bool:
            new_val = (new_val == 'True' or new_val == True)
            self.check_syntax = new_val
            
        elif item == 'topicsToRx':
            # Topics to which we should listen, but
            # whose msgs we are to receive. Check
            # whether empty str or empty array, which
            # means unsubscribe from all topics-to-rx
            # (though not from syntax checker and echo):
            if ((type(new_val) == str and len(new_val) == 0)) or\
                (type(new_val) == list and len(new_val) == 1 and len(new_val[0]) == 0):
                for (indx, topic) in enumerate(self.topics_to_rx):
                    self.unsubscribeFromTopicOrPattern(topic)
                    del self.topics_to_rx[indx]
                return
            # We are given one or more topics. Tolerate
            # comma-separated, space-separated strings, 
            # and also arrays:
            if type(new_val) != list:
                # comma-separated string of topics?
                if OnDemandPublisher.FIND_COMMA_PATTERN.search(new_val) is not None:
                    # Yes, commas found: remove all spaces that might be around the commas:
                    new_val = OnDemandPublisher.FIND_SPACE_PATTERN.sub('', new_val)
                    new_val = new_val.split(',')
                # Else must be space-separated string of topics or just a single word:
                else:
                    # Replace all multi-space areas with single spaces; the strip()
                    # eliminates spaces at start and end:
                    new_val = OnDemandPublisher.FIND_SPACE_PATTERN.sub(' ', new_val).strip()
                    # Get an array of topic strings (works even with singleton topic):
                    new_val = new_val.split(' ')

            # Unsubscribe from all topics that are *not*
            # in the new list of topics:
            for (topic_pos, curr_topic) in enumerate(self.topics_to_rx):
                if curr_topic not in new_val:
                    self.topics_to_rx.pop(topic_pos)
                    self.unsubscribeFromTopicOrPattern(curr_topic)
            
            # Subscribe to any topics in the new-list that
            # we are not already subscribed to:  
            for topic in new_val:
                # Disallow use of reserved topics self.echo_topic
                # and self.syntax_check_topic:
                if topic in (self.echo_topic, self.syntax_check_topic):
                    raise ValueError("Warning: '%s' and '%s' are reserved topic names." % (self.echo_topic, self.syntax_check_topic))
                
                try:
                    # Already subscribed to it?    
                    self.topics_to_rx.index(topic)
                    # At least by our bookkeeping, yes 
                    # (else we would have had the exception.
                    # Ensure we really are:
                    if not self.testBus.subscribedTo(topic):
                        self.subscribeTopicOrPattern(topic)
                    continue
                except ValueError:
                    pass
                # New topic:
                self.subscribeTopicOrPattern(topic)
                self.topics_to_rx.append(topic)
                
        else:
            raise KeyError('Key %s is not in schoolbus tester' % item)
        return new_val

    def subscribeTopicOrPattern(self, topic_or_pattern):
        # If the topic contains an asterisk, then
        # build a regex pattern, rather than passing
        # the topic itself:
        try:
            topic_or_pattern.index('*')
            # Topic is a pattern of topics:
            topic = re.compile(topic_or_pattern)
        except ValueError:
            # Not a pattern, just a regular topic name:
            topic = topic_or_pattern
        self.testBus.subscribeToTopic(topic, functools.partial(self.messageReceiver))

    def unsubscribeFromTopicOrPattern(self, topic_or_pattern):
        # If the topic contains an asterisk, then
        # build a regex pattern, rather than passing
        # the topic itself:
        try:
            topic_or_pattern.index('*')
            # Topic is a pattern of topics:
            topic = re.compile(topic_or_pattern)
        except ValueError:
            # Not a pattern, just a regular topic name:
            topic = topic_or_pattern
        self.testBus.unsubscribeFromTopic(topic)

    
    def createMessage(self, topic=None, msgLen=None, content=None):
        '''
        Returns a BusMessage whose content is the given length,
        and whose topicName is the given topic.
        
        :param topic: topic for which this message is destined.
        :type topic: string
        :param msgLen: desired str length
        :type msgLen: int
        :param content: content of the message. If None, a random content
            of length msgLen will be created
        :return: BusMessage object ready to publish. The context field of the
            message instance will be the MD5 of the content.
        :rtype: BusMessage
        '''
    
        if topic is None:
            topic = self._stream_topic
            
        if msgLen is None:
            msgLen = self.standard_msg_len 
    
        if content is None:    
            content = self.createRandomStr(msgLen)
        
        msg = BusMessage(content=content, topicName=topic, context=hashlib.md5(str(content)).hexdigest())

        return msg
        
    def createRandomStr(self, the_len):
        the_len = int(the_len)
        content = bytearray()
        for _ in range(the_len):
            content.append(random.choice(string.letters))
        return str(content)
                
        
    def sendMessage(self, bus_msg=None):
        '''
        Publish the given BusMessage, or
        the standard message:
        '''
        if bus_msg is None:
            bus_msg = self.standard_oneshot_msg
        self.testBus.publish(bus_msg)
    
    def logInfo(self, msg):
        if self.loglevel >= OnDemandPublisher.LOG_LEVEL_INFO:
            print(str(datetime.datetime.now()) + ' info: ' + msg)

    def logErr(self, msg):
        if self.loglevel >= OnDemandPublisher.LOG_LEVEL_ERR:
            print(str(datetime.datetime.now()) + ' error: ' + msg)

    def logDebug(self, msg):
        if self.loglevel >= OnDemandPublisher.LOG_LEVEL_DEBUG:
            print(str(datetime.datetime.now()) + ' debug: ' + msg)
        
        
    def syntaxCheckReceiver(self, busMsg, context=None):
        '''
        Given a BusMessage instance, determine whether it contains 
        all necessary attributes: ID, and time. This method 
        is a callback for topic specified in module variable
        self.syntaxTopic. Constructs a string that lists any
        errors or warnings, and returns that string as a
        synchronous response.
        
        :param busMsg: the message to be evaluated
        :type busMsg: BusMessage
        :param context: not used
        :type context: <any>
        '''
        errors = []
        
        if busMsg.id is None:
            errors.append('Error: No ID field')
        if type(busMsg.time) != int and type(busMsg.time) != float:
            errors.append("Error: Time must be an int, was '%s'" % busMsg.time)
            try:
                busMsg.isoTime
            except (ValueError, TypeError):
                errors.append("Error: Time value not transformable to ISO time '%s'" % busMsg.time)
        try:
            json.loads(busMsg.content)
        except AttributeError:
            errors.append("Warning: no 'content' field.")
        except ValueError:
            errors.append("Warning: 'content' field present, but not valid JSON.")

        if len(errors) == 0:
            response = 'Message OK'
        else:
            response = self.testBus.makeResponseMsg(busMsg, '; '.join(errors))
        
        self.testBus.publish(response)
        
    def messageReceiver(self, busMsg, context=None):
        '''
        Method that is called with each received message.
        
        :param busMsg: bus message object
        :type busMsg: BusMessage
        :param context: context Python structure, if subscribeToTopic() was
            called with one.uu
        :type context: <any>
        '''
        
        self.mostRecentRxTime = time.time()
        self.printedResetting = False
        
        if self.serve_echo and busMsg.topicName == 'echo':
            # Create a message with the same content as the incoming
            # message, timestamp the new message, and publish it
            # on the response topic (i.e. tmp.<msgId>):
            respMsg = self.testBus.makeResponseMsg(busMsg, busMsg.content)

            # Publish a response:
            self.testBus.publish(respMsg)
            self.numEchoed += 1

            if self.numEchoed % 1000 == 0:
                print('Echoed %d' % self.numEchoed)
                
            if self.numEchoed % 10000 == 0:
                # Messages per second:
                msgs_per_sec = 10000.0 / (time.time() - self.batch_start_time)
                print('Echoing: %f Msgs/sec' % msgs_per_sec)
                self.batch_start_time = time.time()
              

        self.numReceived += 1
        if self.numReceived % 1000 == 0:
            stats_msg = 'Rxed %d' % self.numReceived
            self.logInfo(stats_msg)
            self.bus_stats_queue.put_nowait(stats_msg)
            
        if self.numReceived % 10000 == 0:
            # Messages per second:
            msgs_per_sec = 10000.0 / (time.time() - self.batch_start_time)
            stats_msg1 = 'Rx (no echo): %f Msgs/sec' % msgs_per_sec
            self.logInfo(stats_msg1)
            if not self.bus_stats_queue.full():
                self.bus_stats_queue.put_nowait(stats_msg1)
            self.batch_start_time = time.time()
                        
        topic = busMsg.topicName
        matching_subscribed_topic = self.findTopic(topic, self.topics_to_rx)
        if matching_subscribed_topic is not None:
            if not self.bus_msg_queue.full():
                self.bus_msg_queue.put_nowait(str(datetime.datetime.now()) +\
                                          "--%s: " % matching_subscribed_topic +\
                                          busMsg.content)
    
    def findTopic(self, topicIdentifier, topicStrArr):
        '''
        Given either a string or a pattern, and an
        array of strings that correspond to topic names
        return return the topic name that matches the
        topicIdentifier, or None. The strings in topicStrArr
        may be regular expressions as used in pattern subscriptions.
        In that case a regex match is done against the topicIdentifier.
        
        :param topicIdentifier: a string that names a topic. This is
            a straight name, not a regex
        :type topicIdentifier: string
        :param topicsStrArr: array of strings, some of which may contain
            the wildcard char '*', indicating that the string is a 
            regex pattern. In such a case the topicIdentifier is
            checked against that regex pattern.
        :return: the string in topicsStrArr that matched, or None. 
            The returned string may, of course be a regex string if
            that's what caused a match.
        :rType: {string | None}
        '''
         
        for topic_in_list in topicStrArr:
            # Is it a pattern?
            asterisk_pos = string.find(topic_in_list, '*')
            if asterisk_pos == -1:
                # Simple topic name, check exact match:
                if topicIdentifier == topic_in_list:
                    return topic_in_list
            else:
                # Topic 'name' in the list is actually a pattern:
                if re.match(topic_in_list, topicIdentifier) is not None:
                    return topic_in_list
                
        # No match:
        return None

                       
    def resetEchoedCounter(self):
        currTime = time.time()
        
        if self.mostRecentRxTime is None:
            # Nothing received yet:
            self.startTime = time.time()
            self.batch_start_time = time.time()
            self.startIdleTimer()
            return
        
        if currTime - self.mostRecentRxTime <= OnDemandPublisher.MAX_IDLE_TIME:
            # Received msgs during more recently than idle time:
            self.startIdleTimer() 
            return
        
        # Did not receive msgs within idle time:
        self.printTiming()
        if not self.printedResetting:
            resetMsg = 'Resetting (echoed %d)' % self.numEchoed
            self.logInfo(resetMsg)
            self.bus_stats_queue.put_nowait(resetMsg)
            self.printedResetting = True
        self.numEchoed = 0
        self.startTime = time.time()
        self.batch_start_time = time.time()
        self.timer = self.startIdleTimer()

    def startIdleTimer(self):
        threading.Timer(OnDemandPublisher.MAX_IDLE_TIME, functools.partial(self.resetEchoedCounter)).start()

    def stop(self, signum=None, frame=None):
        try:
            self.logDebug('Unsubscribing from echo...')
            self.serve_echo = False
            self.logDebug('Unsubscribing from check_syntax...')
            self.check_syntax = False
            
            self.logInfo('Shutting down streamer thread...')
            self.msg_streamer.stop(signum=None, frame=None)
            self.msg_streamer.join(JOIN_WAIT_TIME)
            if self.msg_streamer.is_alive():
                raise TimeoutError("Unable to stop message streamer thread '%s'." % self.msg_streamer.name)
            
        except Exception as e:
            print("Error while shutting down message streamer thread: '%s'" % `e`)

        self.done = True
        self.interruptEvent.set()
        
        
    def printTiming(self, startTime=None):
        currTime = time.time()
        if startTime is None:
            startTime = self.startTime
            return
        echoMsg = 'Echoed %d messages' % self.numEchoed
        self.logInfo(echoMsg)
        if self.numEchoed > 0:
            timeElapsed = float(currTime) - float(self.startTime)
            rateMsg = 'Msgs per second: %d' % (timeElapsed / self.numEchoed)
            self.logInfo(rateMsg)
            self.bus_stats_queue.put_nowait(rateMsg)
            
    def run(self):
        while not self.done:
            self.startTime = time.time()
            self.startIdleTimer()
            self.interruptEvent.wait()
            if not self.done and self.sendMsg:
                self.testBus.publish(self.outMsg)
                self.interruptEvent.clear()
                self.sendMsg = False
            continue
        
        self.testBus.unsubscribeFromTopic(self._echo_topic)
        self.testBus.close()
Exemplo n.º 40
0
    def __init__(self,
                 serveEchos=True,
                 listenOn=None,
                 streamMsgs=False,
                 checkSyntax=True,
                 oneShotMsg=None):
        '''
        Initialize the test harness server. 
        
        :param serveEchos: if True receives messages on ECHO_TOPIC, and returns  
            message with the same content, but different timestamp as a synchronous call. 
        :type serveEchos: bool
        :param listenOn: if this parameter provides an array of topic(s), messages on these
            topics are received and counted, then placed on self.bus_msg_queue. Intermittent reception counts
            and reception rates are printed to the console, and place on self.bus_stats_queue.
        :type listenOn: {None | [string]}
        :param streamMsgs: if True, a continuous stream of messages are sent to
            STREAM_TOPIC. The timestamp is changed each time.
        :type streamMsgs: bool
        :param checkSyntax: if true, listens on SYNTAX_TOPIC, checks incoming messages
            for conformance to SchoolBus message format, and synchronously returns a result.
        :type checkSyntax: bool
        :param oneShotMsg: if non-none this parameter is expected to be a two-tuple
            containing a topic and a content. Either may be None. If topic is None,
            the message is sent to STREAM_TOPIC. If content is None, content will be
            a random string of length STANDARD_MSG_LENGTH.
        :type oneShotMsg: { None | ({string | None}, {string | None})}
        '''
        threading.Thread.__init__(self)

        self.loglevel = OnDemandPublisher.LOG_LEVEL_DEBUG
        #self.loglevel = OnDemandPublisher.LOG_LEVEL_INFO
        #self.loglevel = OnDemandPublisher.LOG_LEVEL_NONE

        #***********
        #         print ('serveEchos %s' % serveEchos)
        #         print ('listenOn %s' % listenOn)
        #         print ('streamMsgs %s' % streamMsgs)
        #         print('checkSyntax %s' % checkSyntax)
        #         print('oneShotMsg %s' % str(oneShotMsg))
        #         sys.exit()
        #***********

        self._serveEchos = serveEchos

        self.daemon = True
        self.testBus = BusAdapter()
        self.done = False
        # No topics yet that we are to listen to, but drop msgs for:
        self.topics_to_rx = []

        self.echo_topic = ECHO_TOPIC
        self.syntax_check_topic = SYNTAX_TOPIC
        self.standard_msg_len = STANDARD_MSG_LENGTH
        self._stream_interval = STREAM_INTERVAL

        self._stream_topic = STREAM_TOPIC
        self.standard_bus_msg = self.createMessage(STREAM_TOPIC,
                                                   STANDARD_MSG_LENGTH,
                                                   content=None)

        self.one_shot_topic = STREAM_TOPIC
        self.one_shot_content = self.standard_bus_msg

        # Queue into which incoming messages are fed
        # for consumers to communicated to clients, such
        # as a Web UI:
        self.bus_msg_queue = Queue.Queue(MAX_SAVED_MSGS)

        # Queue to which aggregate stats will be written:
        self.bus_stats_queue = Queue.Queue(MAX_SAVED_MSGS)

        # Create a 'standard' message to send out
        # when asked to via SIGUSR1 or the Web:

        if oneShotMsg is not None:
            # The oneShotMsg arg may have one of the following values:
            #   - None if oneshot was not requested.
            #   - An empty array if -o (or --oneshot)
            #        was listed in the command line,
            #        but neither topic nor message content
            #        were specified.
            #   - A one-element array if the topic was
            #        specified
            #   - A two-element array if both topic and
            #        content were specified.

            (one_shot_topic, self.one_shot_content) = oneShotMsg
            if one_shot_topic is not None:
                self.one_shot_topic = one_shot_topic

            # Create a message that one can have sent on demand
            # by sending a SIGUSR1 to the server, or using
            # the Web UI:

            self.standard_oneshot_msg = self.createMessage(
                self.one_shot_topic, content=self.one_shot_content)
            # Send the standard msg:
            self.sendMessage(self.standard_oneshot_msg)
        else:
            # Create a standard message with default topic/content:
            self.standard_oneshot_msg = self.createMessage()

        # Start time of a 10,000 messages batch:
        self.batch_start_time = time.time()

        # Subscribe, and ensure that context is delivered
        # with each message:

        if serveEchos:
            self.serve_echo = True

        if listenOn is not None:
            # Array of topics to listen to
            # If we are to listen to some (additional) topic(s),
            # on which no action is taken, subscribe to it/them now:
            self['topicsToRx'] = listenOn

        if checkSyntax:
            self.check_syntax = True

        self.interruptEvent = threading.Event()

        # Number of messages received and echoed:
        self.numEchoed = 0
        # Alternative to above when not echoing: Number of msgs just received:
        self.numReceived = 0
        self.mostRecentRxTime = None
        self.printedResetting = False

        # Not asked to send a message yet.
        self.sendMsg = False

        if streamMsgs is not None:
            # The streamMsgs arg may have one of the following values:
            #   - None if streaming messages was not requested.
            #   - An empty array if -s (or --streamMsgs)
            #        was listed in the command line,
            #        but neither topic nor message content
            #        were specified.
            #   - A one-element array if the topic was
            #        specified
            #   - A two-element array if both topic and
            #        content were specified.

            self.msg_streamer = MessageOutStreamer(self.standard_bus_msg)

            # The caller passed in a tuple with
            # topic and content for streaming. If they
            # are non-None, update the streamer:

            (stream_topic, stream_content) = streamMsgs
            if stream_topic is not None:
                self.stream_topic = stream_topic
            if stream_content is not None:
                self.stream_content = stream_content

            # Start streaming:

            self.msg_streamer.pause(True)
            self.msg_streamer.start()

        else:
            # Create a standard message with default topic/content:
            self.standard_stream_msg = self.createMessage()
            self.msg_streamer = None
class MsgDeliveryTest(unittest.TestCase):

    def setUp(self):
        self.msg_server = OnDemandPublisher()
        self.msg_server.start()
        self.deliveryDest = functools.partial(self._deliveryDest)
        self.bus = BusAdapter()
        self.delivery_event = threading.Event()
        
        # Content and topic for outgoing msgs sent
        # via the OnDemandPublisher instance:
        
        self.reference_msg_content = 'rTmzntNSQLmXokesBpLmAbPYeysftXnuntfdPKrxMVNUuqVFHFzfcrrSaRssdHuMRhPYjXKYrJjwKcyeYycEzQSkJubTabeSFLRS'
        self.reference_named_topic = 'MyTopic'
        self.reference_pattern_topic = r'MyTopic*'
        self.reference_pattern_obj = re.compile(self.reference_pattern_topic)
        self.reference_pattern_matching_topic = 'MyTopicHurray'
        self.reference_context = 'myContext'
        
        # Msg with a non-pattern, i.e. fixed-name topic:
        self.reference_bus_msg = BusMessage(topicName=self.reference_named_topic, 
                                            content=self.reference_msg_content, 
                                            context=self.reference_context)
        
        # Msg with a topic that will match self.reference_pattern_topic: 
        self.reference_bus_wildcard_msg = BusMessage(topicName=self.reference_pattern_matching_topic, 
                                                     content=self.reference_msg_content,
                                                     context=self.reference_context) 
        
    def tearDown(self):
        self.msg_server.stop()
        self.msg_server.join()
        self.bus.close()
        self.delivery_event.clear()

    @unittest.skipIf(not TEST_ALL, 'Temporarily disabled')
    def testDeliveryNameNotThreaded(self):
        self.bus.subscribeToTopic(self.reference_named_topic, self.deliveryDest, threaded=False, context=self.reference_context)
        
        self.delivery_event.clear()
        self.msg_server.sendMessage(self.reference_bus_msg)
        
        # Wait for message to arrive:
        self.delivery_event.wait()
        # Compare our expected incoming bus message with
        # what we actually got:
        self.assertBusMsgsEqual(self.reference_bus_msg, self.received_bus_msg)

    @unittest.skipIf(not TEST_ALL, 'Temporarily disabled')
    def testDeliveryNameThreaded(self):
        self.bus.subscribeToTopic(self.reference_named_topic, self.deliveryDest, threaded=True, context=self.reference_context)
        self.delivery_event.clear()
        self.msg_server.sendMessage(self.reference_bus_msg)
        
        # Wait for message to arrive:
        self.delivery_event.wait()
        
        # Compare our expected incoming bus message with
        # what we actually got:
        self.assertBusMsgsEqual(self.reference_bus_msg, self.received_bus_msg)

    @unittest.skipIf(not TEST_ALL, 'Temporarily disabled')
    def testDeliveryPatternNotThreaded(self):
        # Subscribe to a pattern topic, using an re.Pattern instance:
        self.bus.subscribeToTopic(self.reference_pattern_obj, self.deliveryDest, threaded=False, context=self.reference_context)        
        
        self.delivery_event.clear()
        # Send to a topic that fits the reference pattern ('MyTopic*'):
        self.msg_server.sendMessage(self.reference_bus_wildcard_msg)

        self.delivery_event.wait()

        self.assertBusMsgsEqual(self.reference_bus_wildcard_msg, self.received_bus_msg)

    @unittest.skipIf(not TEST_ALL, 'Temporarily disabled')
    def testDeliveryPatternThreaded(self):
        # Subscribe to a pattern topic, using an re.Pattern instance:
        self.bus.subscribeToTopic(self.reference_pattern_obj, self.deliveryDest, threaded=True, context=self.reference_context)        
        
        self.delivery_event.clear()
        # Send to a topic that fits the reference pattern ('MyTopic*'):
        self.msg_server.sendMessage(self.reference_bus_wildcard_msg)

        self.delivery_event.wait()

        self.assertBusMsgsEqual(self.reference_bus_wildcard_msg, self.received_bus_msg)

    @unittest.skipIf(not TEST_ALL, 'Temporarily disabled')
    def testSyncPublish(self):
        
        echo_msg = self.reference_bus_msg
        
        # Change target topic to what the test harness echo server
        # listens for:
        echo_msg.topicName = self.msg_server.ECHO_CHANNEL
        
        result = self.bus.publish(echo_msg, sync=True)
        self.assertEqual(self.reference_msg_content, result)

    # -------------------------  Service Methods --------------

    def assertBusMsgsEqual(self, expected_msg, actual_msg):
        self.assertEqual(expected_msg.topicName, actual_msg.topicName)
        self.assertEqual(expected_msg.context, actual_msg.context)
        self.assertEqual(expected_msg.content, actual_msg.content)

    def _deliveryDest(self, bus_msg):
        '''
        Receives incoming messages, and places them into
        instance variable received_bus_msg.
        
        :param bus_msg:
        :type bus_msg:
        '''
        #print('BusMessage: context is %s' % bus_msg.context)
        self.received_bus_msg = bus_msg
        self.delivery_event.set()