class Sender(object):
__slots__ = ('__conn', '__session')
__exchange = 'amq.topic'
def __init__(self, host='localhost', port=5672, timeout=5):
self.__conn = None
qpid.util.socket.setdefaulttimeout(timeout)
if __debug__:
print 'Connecting to %s:%d' % (host, port)
sock = qpid.util.connect(host, port)
self.__conn = Connection(sock=sock)
self.__conn.start()
self.__session = self.__conn.session(str(uuid4()))
def __del__(self):
if self.__conn: self.__conn.close()
def send(self, routing_key, msg_text):
propDelivery = self.__session.delivery_properties(
routing_key = routing_key.__str__(),
exchange = self.__exchange )
contentType = 'application/x-protobuf'
propMessage = self.__session.message_properties(
content_type = contentType)
self.__session.message_transfer(destination = self.__exchange,
message = Message(propDelivery, propMessage, msg_text))
class Receiver(object):
__slots__ = ('__conn', '__session', '__queue')
__exchange = 'amq.topic'
def __init__(self, host='localhost', port=5672, timeout=5):
self.__conn = self.__session = None
if __debug__:
print 'Connecting to %s:%d' % (host, port)
qpid.util.socket.setdefaulttimeout(timeout)
sock = qpid.util.connect(host, port)
self.__conn = Connection(sock=sock)
self.__conn.start()
self.__session = self.__conn.session(str(uuid4()))
def __del__(self):
if self.__session: self.__session.close()
if self.__conn: self.__conn.close()
def setup(self, binding_keys, queue_name = 'queue_%d' % os.getpid()):
# Setup queue
self.__session.queue_declare(queue = queue_name, auto_delete = True,
exclusive = False)
for k in binding_keys:
if __debug__:
print 'Binding', k
self.__session.exchange_bind(exchange = self.__exchange,
queue = queue_name, binding_key = k)
localQueueName = 'local_queue_%d' % os.getpid()
self.__queue = self.__session.incoming(localQueueName)
# Route messages from message_queue to my_local_queue
self.__session.message_subscribe(queue = queue_name,
destination = localQueueName)
if __debug__:
print 'Queue info:', self.__session.queue_query(queue_name)
self.__queue.start()
return self
def fetch(self, timeout=None):
message = self.__queue.get(timeout=timeout)
self.__session.message_accept(RangedSet(message.id))
return message
def start(self):
"""
Enable AMQP queueing. This method puts up the event processor and
sets it to "active".
"""
self.log.debug("enabling AMQP queueing")
# Evaluate username
user = self.config.get("amqp.id", default=None)
if not user:
user = self.env.uuid
password = self.config.get("amqp.key")
# Create initial broker connection
url = "%s:%s" % (self.url['host'], self.url['port'])
self._conn = Connection.establish(url, reconnect=self.reconnect,
username=user,
password=password,
transport=self.url['transport'],
reconnect_interval=self.reconnect_interval,
reconnect_limit=self.reconnect_limit)
# Do automatic broker failover if requested
if self.config.get('amqp.failover', False):
auto_fetch_reconnect_urls(self._conn)
# Create event exchange
socket = connect(self.url['host'], self.url['port'])
if self.url['scheme'][-1] == 's':
socket = ssl(socket)
user = self.config.get("amqp.id", default=None)
if not user:
user = self.env.uuid
connection = DirectConnection(sock=socket,
username=user,
password=self.config.get("amqp.key"))
connection.start()
session = connection.session(str(uuid4()))
# pylint: disable=E1103
session.exchange_declare(exchange=self.env.domain, type="xml")
connection.close()
# Create event provider
self._eventProvider = EventProvider(self.env, self.getConnection())
work_headers['Arguments'] = '"5"'
work_headers['Iwd'] = '"/tmp"'
work_headers['Owner'] = '"nobody"'
message_props = session.message_properties(application_headers=work_headers)
message_props.reply_to = session.reply_to('amq.direct', replyTo)
message_props.message_id = uuid4()
print 'Job Request Message ID: %s' % str(message_props.message_id)
delivery_props = session.delivery_properties(routing_key='grid', delivery_mode=2)
delivery_props.ttl = 10000
for num in range(0, num_msgs):
session.message_transfer(destination='amq.direct', message=Message(message_props, delivery_props, ''))
message_props.message_id = uuid4()
dump_queue(recv_queue, session, connection, num_msgs, tout, dest, broker_info)
# Close the session before exiting so there are no open threads.
try:
connection.close()
except:
pass
try:
session.close(timeout=10)
except:
pass
return(SUCCESS)
if __name__ == '__main__':
sys.exit(main())
def dump_queue(binfo, queue_name, to):
# Create a client and log in to it.
child_connection = Connection(sock=connect(str(binfo['ip']), int(binfo['port'])))
child_connection.start()
child_session = child_connection.session(str(uuid4()))
child_session.queue_declare(queue=queue_name, exclusive=True)
child_session.exchange_bind(exchange='amq.direct', queue=queue_name, binding_key=queue_name)
print 'Messages queue: ' + queue_name
# Create the local queue. Use the queue name as destination name
dest = queue_name
queue = child_session.incoming(dest)
# Subscribe the local queue to the queue on the server
child_session.message_subscribe(queue=queue_name, destination=dest, accept_mode=child_session.accept_mode.explicit)
child_session.message_flow(dest, child_session.credit_unit.message, 0xFFFFFFFFL)
child_session.message_flow(dest, child_session.credit_unit.byte, 0xFFFFFFFFL)
# Read responses as they come in and print to the screen.
message = 0
count = 0
while True:
try:
message = queue.get(timeout=to)
count = count + 1
if count == 1:
print 'Received first reponse: %s ' % str(time.time())
except Empty:
print 'Received %s messages: %s' % (str(count), str(time.time() - to))
break
# except qpid.session.Closed:
# print "Re-establishing"
# try:
# child_connection.close()
# except:
# pass
#
# # Give broker time to stablize and accept connections
# time.sleep(2)
# child_connection = Connection(sock=connect(str(binfo['ip']), int(binfo['port'])))
# child_connection.start()
# child_session = child_connection.session(str(uuid4()))
# child_session.queue_declare(queue=queue_name, exclusive=True)
# child_session.exchange_bind(exchange='amq.direct', queue=queue_name, binding_key=queue_name)
#
# # Create the local queue. Use the queue name as destination name
# queue = child_session.incoming(dest)
#
# # Subscribe the local queue to the queue on the server
# child_session.message_subscribe(queue=queue_name, destination=dest, accept_mode=child_session.accept_mode.explicit)
# child_session.message_flow(dest, child_session.credit_unit.message, 0xFFFFFFFFL)
# child_session.message_flow(dest, child_session.credit_unit.byte, 0xFFFFFFFFL)
except:
print 'Unexpected exception!'
break
if message != 0:
child_session.message_accept(RangedSet(message.id))
child_session.close(timeout=10)
child_connection.close()
return (0)
class QpidAMQPHubExtension(BaseAMQPHubExtension):
"""
Initialize the Moksha Hub.
`broker`
[amqps://][<user>[/<password>]@]<host>[:<port>]
"""
def __init__(self, hub, config):
self.config = config
self.set_broker(self.config.get('amqp_broker'))
self.socket = connect(self.host, self.port)
if self.url.scheme == URL.AMQPS:
self.socket = ssl(self.socket)
self.connection = Connection(sock=self.socket,
username=self.user,
password=self.password)
self.connection.start()
log.info("Connected to AMQP Broker %s" % self.host)
self.session = self.connection.session(str(uuid4()))
self.local_queues = []
super(QpidAMQPHubExtension, self).__init__()
def set_broker(self, broker):
self.url = URL(broker)
self.user = self.url.password or 'guest'
self.password = self.url.password or 'guest'
self.host = self.url.host
if self.url.scheme == URL.AMQPS:
self.ssl = True
default_port = 5671
else:
self.ssl = False
default_port = 5672
self.port = self.url.port or default_port
def send_message(self, topic, message, **headers):
headers['routing_key'] = headers.get('routing_key', topic)
props = self.session.delivery_properties(**headers)
msg = Message(props, message)
self.session.message_transfer(destination=headers.get(
'exchange', 'amq.topic'),
message=msg)
super(QpidAMQPHubExtension, self).send_message(topic, message,
**headers)
def subscribe_queue(self, server_queue_name, local_queue_name):
queue = self.session.incoming(local_queue_name)
self.session.message_subscribe(queue=server_queue_name,
destination=local_queue_name)
queue.start()
return queue
def queue_declare(self,
queue,
durable=True,
exclusive=False,
auto_delete=False,
**kw):
self.session.queue_declare(queue=queue,
exclusive=exclusive,
auto_delete=auto_delete,
arguments={
'qpid.max_count': 0,
'qpid.max_size': 0
},
**kw)
def exchange_bind(self, queue, exchange='amq.topic', binding_key=None):
self.session.exchange_bind(exchange=exchange,
queue=queue,
binding_key=binding_key)
def message_subscribe(self, queue, destination):
return self.session.message_subscribe(queue=queue,
destination=destination)
def message_accept(self, message):
try:
self.session.message_accept(RangedSet(message.id))
except SessionClosed:
log.debug("Accepted message on closed session: %s" % message.id)
pass
def subscribe(self, topic, callback):
queue_name = '_'.join([
"moksha_consumer",
self.session.name,
str(uuid4()),
])
server_queue_name = local_queue_name = queue_name
self.queue_declare(queue=server_queue_name,
exclusive=True,
auto_delete=True)
self.exchange_bind(server_queue_name, binding_key=topic)
self.local_queues.append(self.session.incoming(local_queue_name))
self.message_subscribe(queue=server_queue_name,
destination=local_queue_name)
self.local_queues[-1].start()
self.local_queues[-1].listen(callback)
super(QpidAMQPHubExtension, self).subscribe(topic, callback)
def close(self):
self.session.close(timeout=2)
self.connection.close(timeout=2)
self.socket.close()
class QpidAMQPHubExtension(BaseAMQPHubExtension):
"""
Initialize the Moksha Hub.
`broker`
[amqps://][<user>[/<password>]@]<host>[:<port>]
"""
def __init__(self, hub, config):
self.config = config
self.set_broker(self.config.get('amqp_broker'))
self.socket = connect(self.host, self.port)
if self.url.scheme == URL.AMQPS:
self.socket = ssl(self.socket)
self.connection = Connection(sock=self.socket,
username=self.user,
password=self.password)
self.connection.start()
log.info("Connected to AMQP Broker %s" % self.host)
self.session = self.connection.session(str(uuid4()))
self.local_queues = []
super(QpidAMQPHubExtension, self).__init__()
def set_broker(self, broker):
self.url = URL(broker)
self.user = self.url.password or 'guest'
self.password = self.url.password or 'guest'
self.host = self.url.host
if self.url.scheme == URL.AMQPS:
self.ssl = True
default_port = 5671
else:
self.ssl = False
default_port = 5672
self.port = self.url.port or default_port
def send_message(self, topic, message, **headers):
headers['routing_key'] = headers.get('routing_key', topic)
props = self.session.delivery_properties(**headers)
msg = Message(props, message)
self.session.message_transfer(
destination=headers.get('exchange', 'amq.topic'),
message=msg)
super(QpidAMQPHubExtension, self).send_message(
topic, message, **headers)
def subscribe_queue(self, server_queue_name, local_queue_name):
queue = self.session.incoming(local_queue_name)
self.session.message_subscribe(queue=server_queue_name,
destination=local_queue_name)
queue.start()
return queue
def queue_declare(self, queue, durable=True, exclusive=False,
auto_delete=False, **kw):
self.session.queue_declare(queue=queue, exclusive=exclusive,
auto_delete=auto_delete,
arguments={'qpid.max_count': 0,
'qpid.max_size': 0}, **kw)
def exchange_bind(self, queue, exchange='amq.topic', binding_key=None):
self.session.exchange_bind(exchange=exchange, queue=queue,
binding_key=binding_key)
def message_subscribe(self, queue, destination):
return self.session.message_subscribe(queue=queue,
destination=destination)
def message_accept(self, message):
try:
self.session.message_accept(RangedSet(message.id))
except SessionClosed:
log.debug("Accepted message on closed session: %s" % message.id)
pass
def subscribe(self, topic, callback):
queue_name = '_'.join([
"moksha_consumer", self.session.name, str(uuid4()),
])
server_queue_name = local_queue_name = queue_name
self.queue_declare(queue=server_queue_name, exclusive=True,
auto_delete=True)
self.exchange_bind(server_queue_name, binding_key=topic)
self.local_queues.append(self.session.incoming(local_queue_name))
self.message_subscribe(queue=server_queue_name,
destination=local_queue_name)
self.local_queues[-1].start()
self.local_queues[-1].listen(callback)
super(QpidAMQPHubExtension, self).subscribe(topic, callback)
def close(self):
self.session.close(timeout=2)
self.connection.close(timeout=2)
self.socket.close()
session.exchange_bind(queue="test-queue", exchange="test", binding_key="key")
session.message_subscribe(queue="test-queue",
destination="consumer_tag",
accept_mode=session.accept_mode.none,
acquire_mode=session.acquire_mode.pre_acquired)
session.message_flow(destination="consumer_tag",
unit=session.credit_unit.message,
value=0xFFFFFFFFL)
session.message_flow(destination="consumer_tag",
unit=session.credit_unit.byte,
value=0xFFFFFFFFL)
queue = session.incoming("consumer_tag")
dp = session.delivery_properties(
routing_key="key",
delivery_mode=session.delivery_mode.non_persistent,
priority=session.delivery_priority.medium)
sent = Message(dp, "Hello World!")
result = session.queue_query(queue="test-queue")
session.message_transfer(destination="test",
message=sent,
accept_mode=session.accept_mode.none,
acquire_mode=session.acquire_mode.pre_acquired)
msg = queue.get(timeout=10)
print(sent.body == msg.body, sent.body, msg.body)
conn.close()
input()
