def consume_messages(amqp_string):
conn = BrokerConnection(amqp_string)
queue = conn.SimpleQueue("eventstream")
while True:
try:
message = queue.get(block=False, timeout=1)
except Empty:
break
else:
message.ack()
pl = message.payload
log_message = {
'state': pl['state'],
'state_msg': pl['state_msg'],
'host': pl['host'],
'body': pl['body'],
'timestamp': pl['timestamp'],
'html': loader.load("message.html").generate(message=pl)
}
MessageHandler.update_cache(log_message)
MessageHandler.send_updates(log_message)
queue.close()
conn.close()
def wait_many(timeout=1):
#: Create connection
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
connection = BrokerConnection("amqp://*****:*****@localhost:5672//")
#: SimpleQueue mimics the interface of the Python Queue module.
#: First argument can either be a queue name or a kombu.Queue object.
#: If a name, then the queue will be declared with the name as the queue
#: name, exchange name and routing key.
queue = connection.SimpleQueue("kombu_demo")
while True:
try:
message = queue.get(block=False, timeout=timeout)
except Empty:
break
else:
spawn(message.ack)
print(message.payload)
queue.close()
connection.close()
def exchange_send(data,exchange):
try:
connection = BrokerConnection()
channel = connection.channel()
producer = Producer(channel, Exchange(exchange, type="fanout"))
producer.publish(data)
channel.close()
connection.close()
except Exception, error:
print(error)
def _establish_connection(self, conn: BrokerConnection) -> None:
"""
We don't use a pool here. We only have one consumer connection per process, so
we get no value from a pool, and we want to use a heartbeat to keep the consumer
collection alive, which does not work with a pool
:return: the connection to the transport
"""
try:
self._logger.debug("Establishing connection.")
self._conn = conn.ensure_connection(max_retries=3)
self._logger.debug('Got connection: %s', conn.as_uri())
except kombu_exceptions.OperationalError as oe:
self._logger.error("Error connecting to RMQ, could not retry %s", oe)
# Try to clean up the mess
if self._conn is not None:
self._conn.close()
else:
conn.close()
def main():
conn = BrokerConnection("amqp://*****:*****@localhost:5672//")
queue = conn.SimpleQueue("eventstream")
queue_options = {'serializer': 'json', 'compression': 'zlib'}
host = socket.gethostname()
while True:
queue.put({
'state': random.choice(bootstrap_labels),
'state_msg': 'Hello',
'host': host,
'timestamp': datetime.datetime.now().isoformat(),
'body': 'foobar'}, **queue_options)
time.sleep(random.random() / 8.0)
queue.close()
conn.close()
def send_many(n):
#: Create connection
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
connection = BrokerConnection("amqp://*****:*****@localhost:5672//")
#: SimpleQueue mimics the interface of the Python Queue module.
#: First argument can either be a queue name or a kombu.Queue object.
#: If a name, then the queue will be declared with the name as the queue
#: name, exchange name and routing key.
queue = connection.SimpleQueue("kombu_demo")
def send_message(i):
queue.put({"hello": "world%s" % (i, )})
pool = eventlet.GreenPool(10)
for i in xrange(n):
pool.spawn(send_message, i)
pool.waitall()
queue.close()
connection.close()
def send_many(n):
#: Create connection
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
connection = BrokerConnection("amqp://*****:*****@localhost:5672//")
#: SimpleQueue mimics the interface of the Python Queue module.
#: First argument can either be a queue name or a kombu.Queue object.
#: If a name, then the queue will be declared with the name as the queue
#: name, exchange name and routing key.
queue = connection.SimpleQueue("kombu_demo")
def send_message(i):
queue.put({"hello": "world%s" % (i, )})
pool = eventlet.GreenPool(10)
for i in xrange(n):
pool.spawn(send_message, i)
pool.waitall()
queue.close()
connection.close()
class test_pika(unittest.TestCase):
def purge(self, names):
chan = self.connection.channel()
map(chan.queue_purge, names)
def setUp(self):
self.connection = BrokerConnection(transport="pika")
try:
self.connection.connect()
except socket.error:
self.connected = False
else:
self.connected = True
self.exchange = Exchange("tamqplib", "direct")
self.queue = Queue("tamqplib", self.exchange, "tamqplib")
def test_produce__consume(self):
if not self.connected:
raise SkipTest("Broker not running.")
chan1 = self.connection.channel()
producer = Producer(chan1, self.exchange)
producer.publish({"foo": "bar"}, routing_key="tamqplib")
chan1.close()
chan2 = self.connection.channel()
consumer = Consumer(chan2, self.queue)
message = consumeN(self.connection, consumer)
self.assertDictEqual(message[0], {"foo": "bar"})
chan2.close()
self.purge(["tamqplib"])
def test_produce__consume_multiple(self):
if not self.connected:
raise SkipTest("Broker not running.")
chan1 = self.connection.channel()
producer = Producer(chan1, self.exchange)
b1 = Queue("pyamqplib.b1", self.exchange, "b1")
b2 = Queue("pyamqplib.b2", self.exchange, "b2")
b3 = Queue("pyamqplib.b3", self.exchange, "b3")
producer.publish("b1", routing_key="b1")
producer.publish("b2", routing_key="b2")
producer.publish("b3", routing_key="b3")
chan1.close()
chan2 = self.connection.channel()
consumer = Consumer(chan2, [b1, b2, b3])
messages = consumeN(self.connection, consumer, 3)
self.assertItemsEqual(messages, ["b1", "b2", "b3"])
chan2.close()
self.purge(["pyamqplib.b1", "pyamqplib.b2", "pyamqplib.b3"])
def test_timeout(self):
if not self.connected:
raise SkipTest("Broker not running.")
chan = self.connection.channel()
self.purge([self.queue.name])
consumer = Consumer(chan, self.queue)
self.assertRaises(socket.timeout, self.connection.drain_events,
timeout=0.3)
consumer.cancel()
def test_basic_get(self):
chan1 = self.connection.channel()
producer = Producer(chan1, self.exchange)
producer.publish({"basic.get": "this"}, routing_key="basic_get")
chan1.close()
chan2 = self.connection.channel()
queue = Queue("amqplib_basic_get", self.exchange, "basic_get")
queue = queue(chan2)
queue.declare()
for i in range(50):
m = queue.get()
if m:
break
time.sleep(0.1)
self.assertEqual(m.payload, {"basic.get": "this"})
chan2.close()
def tearDown(self):
if self.connected:
self.connection.close()
class Connection:
connection = None
def __init__(self, url):
self.url = url
self.__connection = None
self.__running = True
self.channel = None
self.sleep_time = 10
self.reconnect(url)
@staticmethod
def get_instance():
return Connection.connection
def __connect(self):
self.__connection = BrokerConnection(self.url)
self.channel = self.get_channel()
self.rpc_factory = rpc.RpcFactory(self.channel)
self.publisher_factory = publisher.PublisherFactory(self.channel)
self.consumer_factory = consumer.ConsumerFactory(self.channel)
self.__running = True
Connection.connection = self
def get_broker_connection(self):
if self.__connection is None:
self.reconnect(self.url)
return self.__connection
def get_channel(self):
if self.channel is None:
self.channel = self.get_new_channel()
return self.channel
def get_new_channel(self):
if self.__connection is None:
self.reconnect(self.url)
return self.__connection.channel()
def get_rpc_factory(self):
return self.rpc_factory
def reconnect(self, url=None):
cc.acquire()
if self.__connection is not None:
self.release()
if url is not None:
self.url = url
logger.debug("reconnect connection")
attempt = 0
while True:
try:
self.__connect()
cc.release()
return
except Exception as e:
logging.exception(e)
logging.debug("retry again in %s s" % self.sleep_time)
time.sleep(self.sleep_time)
cc.release()
def drain_events(self):
self.__connection.drain_events()
def release(self):
Connection.connection = None
self.__running = False
self.__connection.release()
self.__connection.close()
self.channel = None
self.__connection = None
connection = BrokerConnection(
hostname='rh2.dev.novagile.fr',
transport="redis",
virtual_host=0,
port=6379)
print "Connection Producer to Redis"
connection.connect()
queue = connection.SimpleQueue("pdf_to_jpg")
for f in files:
# open as binary
my_file = open(f, "rb")
my_file.seek(0)
my_file_bcontent = my_file.read()
my_file.close()
# Push !
queue.put({"file_content": my_file_bcontent,
"file_name": f,
"hostname": gethostname(),
"timestamp": time()},
serializer="pickle",
compression=None)
my_file.close()
print "Pushed on queue pdf_to_jpg"
connection.close()
class SimpleBase(TestCase):
abstract = True
def Queue(self, name, *args, **kwargs):
q = name
if not isinstance(q, Queue):
q = self.__class__.__name__
if name:
q = "%s.%s" % (q, name)
return self._Queue(q, *args, **kwargs)
def _Queue(self, *args, **kwargs):
raise NotImplementedError()
def setUp(self):
if not self.abstract:
self.connection = BrokerConnection(transport="memory")
self.q = self.Queue(None, no_ack=True)
def tearDown(self):
if not self.abstract:
self.q.close()
self.connection.close()
def test_produce__consume(self):
if self.abstract:
return
q = self.Queue("test_produce__consume", no_ack=True)
q.put({"hello": "Simple"})
self.assertEqual(q.get(timeout=1).payload, {"hello": "Simple"})
with self.assertRaises(Empty):
q.get(timeout=0.1)
def test_produce__basic_get(self):
if self.abstract:
return
q = self.Queue("test_produce__basic_get", no_ack=True)
q.put({"hello": "SimpleSync"})
self.assertEqual(q.get_nowait().payload, {"hello": "SimpleSync"})
with self.assertRaises(Empty):
q.get_nowait()
q.put({"hello": "SimpleSync"})
self.assertEqual(q.get(block=False).payload, {"hello": "SimpleSync"})
with self.assertRaises(Empty):
q.get(block=False)
def test_clear(self):
if self.abstract:
return
q = self.Queue("test_clear", no_ack=True)
for i in range(10):
q.put({"hello": "SimplePurge%d" % (i, )})
self.assertEqual(q.clear(), 10)
def test_enter_exit(self):
if self.abstract:
return
q = self.Queue("test_enter_exit")
q.close = Mock()
self.assertIs(q.__enter__(), q)
q.__exit__()
q.close.assert_called_with()
def test_qsize(self):
if self.abstract:
return
q = self.Queue("test_clear", no_ack=True)
for i in range(10):
q.put({"hello": "SimplePurge%d" % (i, )})
self.assertEqual(q.qsize(), 10)
self.assertEqual(len(q), 10)
def test_autoclose(self):
if self.abstract:
return
channel = self.connection.channel()
q = self.Queue("test_autoclose", no_ack=True, channel=channel)
q.close()
def test_custom_Queue(self):
if self.abstract:
return
n = self.__class__.__name__
exchange = Exchange("%s-test.custom.Queue" % (n, ))
queue = Queue("%s-test.custom.Queue" % (n, ),
exchange,
"my.routing.key")
q = self.Queue(queue)
self.assertEqual(q.consumer.queues[0], queue)
q.close()
def test_bool(self):
if self.abstract:
return
q = self.Queue("test_nonzero")
self.assertTrue(q)
"""
Example that sends a single message and exits using the simple interface.
You can use `simple_receive.py` (or `complete_receive.py`) to receive the
message sent.
"""
from kombu import BrokerConnection
#: Create connection
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
connection = BrokerConnection(hostname="localhost",
userid="guest",
password="******",
virtual_host="/")
#: SimpleQueue mimics the interface of the Python Queue module.
#: First argument can either be a queue name or a kombu.Queue object.
#: If a name, then the queue will be declared with the name as the queue
#: name, exchange name and routing key.
queue = connection.SimpleQueue("kombu_demo")
queue.put({"hello": "world"}, serializer="json", compression="zlib")
# Always remember to close channels and connections.
queue.close()
connection.close()
class Server(object):
"""
This Server class is used to provide an RPC server
:keyword server_id: Id of the server
:keyword amqp_host: The host of where the AMQP Broker is running.
:keyword amqp_user: The username for the AMQP Broker.
:keyword amqp_password: The password for the AMQP Broker.
:keyword amqp_vhost: The virtual host of the AMQP Broker.
:keyword amqp_port: The port of the AMQP Broker.
:keyword ssl: Use SSL connection for the AMQP Broker.
:keyword threaded: Use of multithreading. If set to true RPC call-execution
will processed parallel (one thread per call) which dramatically improves
performance.
"""
def __init__(self,
server_id,
amqp_host='localhost',
amqp_user ='******',
amqp_password='******',
amqp_vhost='/',
amqp_port=5672,
ssl=False,
threaded=False):
self.logger = logging.getLogger('callme.server')
self.logger.debug('Server ID: %s' % server_id)
self.server_id = server_id
self.threaded = threaded
self.do_run = True
self.is_stopped = True
self.func_dict={}
self.result_queue = queue.Queue()
target_exchange = Exchange("server_"+server_id+"_ex", "direct", durable=False,
auto_delete=True)
self.target_queue = Queue("server_"+server_id+"_queue", exchange=target_exchange,
auto_delete=True, durable=False)
self.connection = BrokerConnection(hostname=amqp_host,
userid=amqp_user,
password=amqp_password,
virtual_host=amqp_vhost,
port=amqp_port,
ssl=ssl)
try:
self.connection.connect()
except IOError:
self.logger.critical("Connection Error: Probably AMQP User has not enough permissions")
raise ConnectionError("Connection Error: Probably AMQP User has not enough permissions")
self.channel = self.connection.channel()
self.publish_connection = BrokerConnection(hostname=amqp_host,
userid=amqp_user,
password=amqp_password,
virtual_host=amqp_vhost,
port=amqp_port,
ssl=ssl)
self.publish_channel = self.publish_connection.channel()
# consume
self.consumer = Consumer(self.channel, self.target_queue)
if self.threaded == True:
self.consumer.register_callback(self._on_request_threaded)
else:
self.consumer.register_callback(self._on_request)
self.consumer.consume()
self.logger.debug('Init done')
def _on_request(self, body, message):
"""
This method is automatically called when a request is incoming. It
processes the incomming rpc calls in a serial manner (no multithreading)
:param body: the body of the amqp message already unpickled by kombu
:param message: the plain amqp kombu.message with aditional information
"""
self.logger.debug('Got Request')
rpc_req = body
if not isinstance(rpc_req, RpcRequest):
self.logger.debug('Not an RpcRequest Instance')
return
self.logger.debug('Call func on Server %s' %self.server_id)
try:
self.logger.debug('corr_id: %s' % message.properties['correlation_id'])
self.logger.debug('Call func with args %s' % repr(rpc_req.func_args))
result = self.func_dict[rpc_req.func_name](*rpc_req.func_args)
self.logger.debug('Result: %s' % repr(result))
self.logger.debug('Build respnse')
rpc_resp = RpcResponse(result)
except Exception as e:
self.logger.debug('exception happened')
rpc_resp = RpcResponse(e, exception_raised=True)
message.ack()
self.logger.debug('Publish respnse')
# producer
src_exchange = Exchange(message.properties['reply_to'], "direct", durable=False,
auto_delete=True)
self.producer = Producer(self.publish_channel, src_exchange, auto_declare=False)
self.producer.publish(rpc_resp, serializer="pickle",
correlation_id=message.properties['correlation_id'])
self.logger.debug('acknowledge')
def _on_request_threaded(self, body, message):
"""
This method is automatically called when a request is incoming and
`threaded` set to `True`. It processes the incomming rpc calls in
a parallel manner (one thread for each request). A seperate Publisher
thread is used to send back the results.
:param body: the body of the amqp message already unpickled by kombu
:param message: the plain amqp kombu.message with aditional information
"""
self.logger.debug('Got Request')
rpc_req = body
if not isinstance(rpc_req, RpcRequest):
self.logger.debug('Not an RpcRequest Instance')
return
message.ack()
self.logger.debug('acknowledge')
def exec_func(body, message, result_queue):
self.logger.debug('Call func on Server %s' %self.server_id)
try:
self.logger.debug('corr_id: %s' % message.properties['correlation_id'])
self.logger.debug('Call func with args %s' % repr(rpc_req.func_args))
result = self.func_dict[rpc_req.func_name](*rpc_req.func_args)
self.logger.debug('Result: %s' % repr(result))
self.logger.debug('Build respnse')
rpc_resp = RpcResponse(result)
except Exception as e:
self.logger.debug('exception happened')
rpc_resp = RpcResponse(e, exception_raised=True)
result_queue.put(ResultSet(rpc_resp,
message.properties['correlation_id'],
message.properties['reply_to']))
p = Thread(target=exec_func,
name=message.properties['correlation_id'],
args=(body, message, self.result_queue))
p.start()
def register_function(self, func, name):
"""
Registers a function as rpc function so that is accessible from the
proxy.
:param func: The function we want to provide as rpc method
:param name: The name with which the function is visible to the clients
"""
self.func_dict[name] = func
def start(self):
"""
Starts the server. If `threaded` is `True` also starts the Publisher
thread.
"""
self.is_stopped = False
if self.threaded == True:
self.pub_thread = Publisher(self.result_queue, self.publish_channel)
self.pub_thread.start()
while self.do_run:
try:
self.logger.debug("drain_events: %s" % repr(self.do_run))
self.connection.drain_events(timeout=1)
except socket.timeout:
self.logger.debug("do_run: %s" % repr(self.do_run))
except:
self.logger.debug("interrupt exception" )
if self.threaded == True:
self.pub_thread.stop()
self.consumer.cancel()
self.connection.close()
self.publish_connection.close()
self.is_stopped = True
return
if self.threaded == True:
self.pub_thread.stop()
self.logger.debug("Normal Exit" )
self.consumer.cancel()
self.connection.close()
self.publish_connection.close()
self.logger.debug("All closed" )
self.is_stopped = True
def stop(self):
"""
Stops the server.
"""
self.logger.debug('Stop server')
self.do_run = False
while not self.is_stopped:
self.logger.debug('wait for stop')
sleep(0.1)
