示例#1
0
    def __init__(self, transport, codec=Codec()):
        self.__log = logging.getLogger("Broker")
        threading.Thread.__init__(self)
        self.__minimal = Minimal(codec=codec, transport=transport)
        self.__rqueue = Queue(1)
        self.__wqueue = Queue(1)
        self.__callback_lock = threading.RLock()

        self.__callbacks = {
            'TOPIC': {},
            'QUEUE': {},
            'POLL': {},
            'PONG': {},
        }

        def __rloop():
            while True:
                self.__log.debug('__rloop reading')
                message = self.__minimal.receive()
                self.__log.debug('__rloop read %r', message)
                self.__rqueue.put(message)

        def __wloop():
            while True:
                self.__log.debug('__wloop reading')
                message = self.__wqueue.get()
                self.__log.debug('__wloop read %r', message)
                self.__minimal.send(message)
                self.__log.debug('__wloop sent %r', message)
                self.__wqueue.task_done()

        self.__rthread = threading.Thread(target=__rloop, name='broker_r')
        self.__rthread.setDaemon(True)

        self.__wthread = threading.Thread(target=__wloop, name='broker_w')
        self.__wthread.setDaemon(True)

        self.setDaemon(True)

        #start threads
        self.__log.debug('Starting threads')
        self.__rthread.start()
        self.__wthread.start()
        self.__log.debug('Threads started')
示例#2
0
    def __init__(self, transport, codec=Codec()):
        self.__log = logging.getLogger("Broker")
        threading.Thread.__init__(self)
        self.__minimal = Minimal(codec=codec, transport=transport)
        self.__rqueue = Queue(1)
        self.__wqueue = Queue(1)
        self.__callback_lock = threading.RLock()

        self.__callbacks = {
            'TOPIC' : {},
            'QUEUE' : {},
            'POLL'  : {},
            'PONG'  : {},
        }

        def __rloop():
            while True:
                self.__log.debug('__rloop reading')
                message = self.__minimal.receive()
                self.__log.debug('__rloop read %r', message)
                self.__rqueue.put(message)

        def __wloop():
            while True:
                self.__log.debug('__wloop reading')
                message = self.__wqueue.get()
                self.__log.debug('__wloop read %r', message)
                self.__minimal.send(message)
                self.__log.debug('__wloop sent %r', message)
                self.__wqueue.task_done()
        
        self.__rthread = threading.Thread(target=__rloop, name='broker_r')
        self.__rthread.setDaemon(True)

        self.__wthread = threading.Thread(target=__wloop, name='broker_w')
        self.__wthread.setDaemon(True)

        self.setDaemon(True)
        
        #start threads
        self.__log.debug('Starting threads')
        self.__rthread.start()
        self.__wthread.start()
        self.__log.debug('Threads started')
#!/usr/bin/env python

from Broker.Messages import Message, Subscribe, Acknowledge
from Broker.Transport import TCP, UDP
from Broker.Codecs import Codec #auto codec selection (thrift or protobuf if thrift isn't installed)
from Broker.Clients import Minimal

server='broker.labs.sapo.pt'
destination = '.*'
destination_type = 'TOPIC'
N=10000

broker = Minimal(codec=Codec(), transport=TCP(host=server))

broker.send(Subscribe(destination=destination, destination_type=destination_type))
for n in xrange(N):
    message = broker.receive()
    broker.send(Acknowledge(message_id=message.message.id, destination=message.destination))
    print message.message.payload
#!/usr/bin/env python

from Broker.Messages import Message, Publish, Authentication
from Broker.Transport import TCP, UDP, SSL
from Broker.Codecs import Protobuf as Codec  #auto codec selection (thrift or protobuf if thrift isn't installed)
from Broker.Clients import Minimal

server = '127.0.0.1'
destination = '/python/tests'
destination_type = 'QUEUE'
N = 1000

broker = Minimal(codec=Codec(), transport=SSL(host=server))

broker.send(
    Authentication.from_sts_credentials(username='******',
                                        password='******'))

for n in xrange(N):
    message = Message(payload='Message number %d' % n)
    publish = Publish(destination=destination,
                      destination_type=destination_type,
                      message=message)
    broker.send(publish)
示例#5
0
class Client(threading.Thread):
    '''Basic client for Broker'''
    def __init__(self, transport, codec=Codec()):
        self.__log = logging.getLogger("Broker")
        threading.Thread.__init__(self)
        self.__minimal = Minimal(codec=codec, transport=transport)
        self.__rqueue = Queue(1)
        self.__wqueue = Queue(1)
        self.__callback_lock = threading.RLock()

        self.__callbacks = {
            'TOPIC': {},
            'QUEUE': {},
            'POLL': {},
            'PONG': {},
        }

        def __rloop():
            while True:
                self.__log.debug('__rloop reading')
                message = self.__minimal.receive()
                self.__log.debug('__rloop read %r', message)
                self.__rqueue.put(message)

        def __wloop():
            while True:
                self.__log.debug('__wloop reading')
                message = self.__wqueue.get()
                self.__log.debug('__wloop read %r', message)
                self.__minimal.send(message)
                self.__log.debug('__wloop sent %r', message)
                self.__wqueue.task_done()

        self.__rthread = threading.Thread(target=__rloop, name='broker_r')
        self.__rthread.setDaemon(True)

        self.__wthread = threading.Thread(target=__wloop, name='broker_w')
        self.__wthread.setDaemon(True)

        self.setDaemon(True)

        #start threads
        self.__log.debug('Starting threads')
        self.__rthread.start()
        self.__wthread.start()
        self.__log.debug('Threads started')

    def send(self, message, block=True, timeout=None):
        return self.__wqueue.put(message, block, timeout)

    def receive(self, block=True, timeout=None):
        msg = self.__rqueue.get(block, timeout)
        self.__rqueue.task_done()
        return msg

    def subscribe(self,
                  destination,
                  callback,
                  destination_type='TOPIC',
                  auto_acknowledge=True):
        self.__log.debug('subscribe(%r, %r, %r, %r)', destination, callback,
                         destination_type, auto_acknowledge)

        with self.__callback_lock:
            #can't resubscribe a topic
            assert (destination not in self.__callbacks[destination_type])
            #can't subscribe a polled destination
            assert (destination not in self.__callbacks['POLL'])

            if 'QUEUE' == destination_type and auto_acknowledge:
                #I hate python's scope semantics
                orig_callback = callback

                def auto_ack_callback(msg):
                    #should I ack before or after processing?
                    self.acknowledge(msg)
                    orig_callback(msg)

                callback = auto_ack_callback

            self.__callbacks[destination_type][destination] = callback

        return self.send(
            Subscribe(destination=destination,
                      destination_type=destination_type))

    #TODO implement timeout
    def poll(self, destination, callback, auto_acknowledge=True):
        with self.__callback_lock:
            #can't poll a subscribed destination
            assert (destination not in self.__callbacks['QUEUE'])

            if auto_acknowledge:
                orig_callback = callback

                def auto_ack_callback(msg):
                    #should I ack before or after processing?
                    self.acknowledge(msg)
                    orig_callback(msg)

                callback = auto_ack_callback

            polls = self.__callbacks['POLL']
            if destination in polls:
                polls[destination].append(callback)
            else:
                polls[destination] = collections.deque([callback])

        return self.send(Poll(destination=destination, timeout=1))

    def unsubscribe(self, destination, destination_type='TOPIC'):
        self.__log.debug('subscribe(%r, %r)', destination, destination_type)

        with self.__callback_lock:
            del self.__callbacks[destination_type][destination]

        return self.send(Unsubscribe(destination, destination_type))

    def publish(self,
                payload,
                destination,
                destination_type='TOPIC',
                expiration=None):
        message = Message(payload=payload, expiration=expiration)

        self.__log.debug('publish(%r, %r, %r, %r)', message, destination,
                         destination_type, expiration)
        return self.send(Publish(message, destination, destination_type))

    def acknowledge(self, message):
        self.__log.debug('acknowledge(%r)', message)
        return self.send(
            Acknowledge(message_id=message.message.id,
                        destination=message.destination))

    #TODO implement timeout
    def ping(self, callback):

        action_id = str(uuid())

        with self.__callback_lock:
            self.__callbacks['PONG'][action_id] = callback

        return self.send(Ping(action_id))

    #TODO take care of timeouts
    def __dispatch_fault(self, msg):
        raise msg

    def __dispatch_pong(self, msg):
        action_id = msg.action_id

        try:
            with self.__callback_lock:
                callback = self.__callbacks['PONG'][action_id]
                del self.__callbacks['PONG'][action_id]

        except KeyError:
            raise KeyError('Unknown pong callbackk for %r' % action_id)

        callback()

    def __dispatch_accepted(self, msg):
        raise NotImplementedError
        pass

    def handle_fault(self, fault):
        self.__log.error('Broker Fault: %r', fault)

    def __dispatch_notification(self, msg):
        try:
            with self.__callback_lock:
                if 'TOPIC' == msg.destination_type:
                    callback = self.__callbacks['TOPIC'][msg.subscription]
                elif 'QUEUE' == msg.destination_type:
                    if msg.subscription in self.__callbacks['QUEUE']:
                        callback = self.__callbacks['QUEUE'][msg.subscription]
                    else:
                        callback = self.__callbacks['POLL'][
                            msg.subscription].popleft()

        except:
            raise KeyError('No handler for %s:%s' %
                           (msg.destination_type, msg.subscription))

        return callback(msg)

    def run_once(self, block=True):
        self.__log.debug('loop_read_once(block=%r)', block)
        msg = self.receive(block)
        self.__log.debug('read message %r', msg)

        #dispatch message types
        try:
            return {
                Notification: self.__dispatch_notification,
                Fault: self.__dispatch_fault,
                Pong: self.__dispatch_pong,
                Accepted: self.__dispatch_accepted,
                Fault: self.handle_fault
            }[msg.__class__](msg)

        except KeyError:
            self.__log.error('Unknown message type %r', msg)
            raise

    def run(self):
        while True:
            self.run_once()
示例#6
0
class Client(threading.Thread):
    '''Basic client for Broker'''

    def __init__(self, transport, codec=Codec()):
        self.__log = logging.getLogger("Broker")
        threading.Thread.__init__(self)
        self.__minimal = Minimal(codec=codec, transport=transport)
        self.__rqueue = Queue(1)
        self.__wqueue = Queue(1)
        self.__callback_lock = threading.RLock()

        self.__callbacks = {
            'TOPIC' : {},
            'QUEUE' : {},
            'POLL'  : {},
            'PONG'  : {},
        }

        def __rloop():
            while True:
                self.__log.debug('__rloop reading')
                message = self.__minimal.receive()
                self.__log.debug('__rloop read %r', message)
                self.__rqueue.put(message)

        def __wloop():
            while True:
                self.__log.debug('__wloop reading')
                message = self.__wqueue.get()
                self.__log.debug('__wloop read %r', message)
                self.__minimal.send(message)
                self.__log.debug('__wloop sent %r', message)
                self.__wqueue.task_done()
        
        self.__rthread = threading.Thread(target=__rloop, name='broker_r')
        self.__rthread.setDaemon(True)

        self.__wthread = threading.Thread(target=__wloop, name='broker_w')
        self.__wthread.setDaemon(True)

        self.setDaemon(True)
        
        #start threads
        self.__log.debug('Starting threads')
        self.__rthread.start()
        self.__wthread.start()
        self.__log.debug('Threads started')

    def send(self, message, block=True, timeout=None):
        return self.__wqueue.put(message, block, timeout)

    def receive(self, block=True, timeout=None):
        msg = self.__rqueue.get(block=True, timeout=None)
        self.__rqueue.task_done()
        return msg

    def subscribe(self, destination, callback, destination_type='TOPIC', auto_acknowledge=True):
        self.__log.debug('subscribe(%r, %r, %r, %r)', destination, callback, destination_type, auto_acknowledge)
        
        with self.__callback_lock:
            #can't resubscribe a topic
            assert(destination not in self.__callbacks[destination_type])
            #can't subscribe a polled destination
            assert(destination not in self.__callbacks['POLL'])

            if 'QUEUE' == destination_type and auto_acknowledge:
                #I hate python's scope semantics
                orig_callback = callback
                def auto_ack_callback(msg):
                    #should I ack before or after processing?
                    self.acknowledge(msg)
                    orig_callback(msg)

                callback = auto_ack_callback

            self.__callbacks[destination_type][destination] = callback

        return self.send(Subscribe(destination=destination, destination_type=destination_type))

    #TODO implement timeout
    def poll(self, destination, callback, auto_acknowledge=True):
        with self.__callback_lock:
            #can't poll a subscribed destination
            assert(destination not in self.__callbacks['QUEUE'])
            
            if auto_acknowledge:
                orig_callback = callback
                def auto_ack_callback(msg):
                    #should I ack before or after processing?
                    self.acknowledge(msg)
                    orig_callback(msg)
                callback = auto_ack_callback
            
            polls = self.__callbacks['POLL']
            if destination in polls:
                polls[destination].append(callback)
            else:
                polls[destination] = collections.deque([callback])
    
        return self.send(Poll(destination=destination, timeout=1))

    def unsubscribe(self, destination, destination_type='TOPIC'):
        self.__log.debug('subscribe(%r, %r)', destination, destination_type)

        with self.__callback_lock:
            del self.__callbacks[destination_type][destination]

        return self.send(Unsubscribe(destination, destination_type))

    def publish(self, payload, destination, destination_type='TOPIC', expiration=None):
        message = Message(payload=payload, expiration=expiration)

        self.__log.debug('publish(%r, %r, %r, %r)', message, destination, destination_type, expiration)
        return self.send(Publish(message, destination, destination_type))

    def acknowledge(self, message):
        self.__log.debug('acknowledge(%r)', message)
        return self.send(Acknowledge(message_id=message.message.id, destination=message.destination))

    #TODO implement timeout
    def ping(self, callback):

        action_id = str(uuid())

        with self.__callback_lock:
            self.__callbacks['PONG'][action_id] = callback

        return self.send(Ping(action_id))

    #TODO take care of timeouts
    def __dispatch_fault(self, msg):
        raise msg

    def __dispatch_pong(self, msg):
        action_id = msg.action_id

        try:
            with self.__callback_lock:
                callback = self.__callbacks['PONG'][action_id]
                del self.__callbacks['PONG'][action_id]

        except KeyError:
            raise KeyError('Unknown pong callbackk for %r' % action_id)

        callback()

    def __dispatch_accepted(self, msg):
        raise NotImplementedError
        pass

    def handle_fault(self, fault):
        self.__log.error('Broker Fault: %r', fault)

    def __dispatch_notification(self, msg):
        try:
            with self.__callback_lock:
                if 'TOPIC' == msg.destination_type:
                    callback = self.__callbacks['TOPIC'][msg.subscription]
                elif 'QUEUE' == msg.destination_type:
                    if msg.subscription in self.__callbacks['QUEUE']:
                        callback = self.__callbacks['QUEUE'][msg.subscription]
                    else:
                        callback = self.__callbacks['POLL'][msg.subscription].popleft()

        except:
            raise KeyError('No handler for %s:%s' % (msg.destination_type, msg.subscription))
    
        return callback(msg)

    def run_once(self, block=True):
        self.__log.debug('loop_read_once(block=%r)', block)
        msg = self.receive(block)
        self.__log.debug('read message %r', msg)

        #dispatch message types
        try:
            return {
                Notification: self.__dispatch_notification,
                Fault:        self.__dispatch_fault,
                Pong:         self.__dispatch_pong,
                Accepted:     self.__dispatch_accepted,
                Fault:        self.handle_fault
            }[msg.__class__](msg)

        except KeyError:
            self.__log.error('Unknown message type %r', msg)
            raise

    def run(self):
        while True:
            self.run_once()
#!/usr/bin/env python

from Broker.Messages import Message, Subscribe, Acknowledge
from Broker.Transport import TCP, UDP
from Broker.Codecs import Codec  #auto codec selection (thrift or protobuf if thrift isn't installed)
from Broker.Clients import Minimal

server = 'broker.labs.sapo.pt'
destination = '.*'
destination_type = 'TOPIC'
N = 10000

broker = Minimal(codec=Codec(), transport=TCP(host=server))

broker.send(
    Subscribe(destination=destination, destination_type=destination_type))
for n in xrange(N):
    message = broker.receive()
    broker.send(
        Acknowledge(message_id=message.message.id,
                    destination=message.destination))
    print message.message.payload
#!/usr/bin/env python

from Broker.Messages import Message, Poll, Acknowledge
from Broker.Transport import TCP, UDP
from Broker.Codecs import Codec  #auto codec selection (thrift or protobuf if thrift isn't installed)
from Broker.Clients import Minimal

server = 'broker.bk.sapo.pt'
destination = '/python/tests/expiration'
N = 10000

broker = Minimal(codec=Codec(), transport=TCP(host=server))

for n in xrange(N):
    broker.send(Poll(destination=destination, timeout=0))
    message = broker.receive()
    broker.send(
        Acknowledge(message_id=message.message.id,
                    destination=message.subscription))
    print message.message.payload
示例#9
0
#!/usr/bin/env python

from Broker.Messages import Message, Poll, Acknowledge
from Broker.Transport import TCP, UDP
from Broker.Codecs import Codec  # auto codec selection (thrift or protobuf if thrift isn't installed)
from Broker.Clients import Minimal

server = "broker.bk.sapo.pt"
destination = "/python/tests/expiration"
N = 10000

broker = Minimal(codec=Codec(), transport=TCP(host=server))

for n in xrange(N):
    broker.send(Poll(destination=destination, timeout=0))
    message = broker.receive()
    broker.send(Acknowledge(message_id=message.message.id, destination=message.subscription))
    print message.message.payload
示例#10
0
#!/usr/bin/env python

from Broker.Messages import Message, Authentication, Poll, Acknowledge
from Broker.Transport import TCP, UDP, SSL
from Broker.Codecs import Codec
from Broker.Clients import Minimal

server='broker.bk.sapo.pt'
destination = '/perl/tests'
destination_type = 'QUEUE'
N=1000

broker = Minimal(codec=Codec(), transport=SSL(host=server))

broker.send(Authentication.from_sts_credentials(username='******', password='******'))

for n in xrange(N):
    broker.send(Poll(destination=destination, timeout=0))
    message = broker.receive()
    print message
    broker.send(Acknowledge(message_id=message.message.id, destination=message.subscription))
    
#!/usr/bin/env python

from Broker.Messages import Message, Publish
from Broker.Transport import TCP, UDP, SSL
from Broker.Codecs import Codec #auto codec selection (thrift or protobuf if thrift isn't installed)
from Broker.Clients import Minimal

from datetime import datetime, timedelta

delta = timedelta(seconds=30)

server='broker.bk.sapo.pt'
destination = '/python/tests/expiration'
destination_type = 'QUEUE'
N=10000

broker = Minimal(codec=Codec(), transport=TCP(host=server))

for n in xrange(N):
    payload = payload='Message number %d' % n
    message = Message(payload=payload, expiration=datetime.utcnow()+delta)
    raw_input('produce %r' % (payload,))
    publish = Publish(destination=destination, destination_type=destination_type, message=message)
    broker.send(publish)
#!/usr/bin/env python

from Broker.Messages import Message, Authentication, Poll, Acknowledge
from Broker.Transport import TCP, UDP, SSL
from Broker.Codecs import Codec
from Broker.Clients import Minimal

server = 'localhost'
destination = '/perl/tests'
destination_type = 'QUEUE'
N = 1000

broker = Minimal(codec=Codec(), transport=SSL(host=server))

broker.send(
    Authentication.from_sts_credentials(username='******',
                                        password='******'))

for n in xrange(N):
    broker.send(Poll(destination=destination, timeout=0))
    message = broker.receive()
    print message
    broker.send(
        Acknowledge(message_id=message.message.id,
                    destination=message.destination))
示例#13
0
#!/usr/bin/env python

from Broker.Messages import Message, Publish, Authentication
from Broker.Transport import TCP, UDP, SSL
from Broker.Codecs import Protobuf as Codec #auto codec selection (thrift or protobuf if thrift isn't installed)
from Broker.Clients import Minimal

server='127.0.0.1'
destination = '/python/tests'
destination_type = 'QUEUE'
N=1000

broker = Minimal(codec=Codec(), transport=SSL(host=server))

broker.send(Authentication.from_sts_credentials(username='******', password='******'))

for n in xrange(N):
    message = Message(payload='Message number %d' % n)
    publish = Publish(destination=destination, destination_type=destination_type, message=message)
    broker.send(publish)