def blocking_submitter(parameters, data_queue, properties=DEFAULT_PROPERTIES):
while True:
try:
connection = BlockingConnection(parameters)
channel = connection.channel()
channel.queue_declare(queue='sandbox', durable=True)
except Exception:
log.error('connection failure', exc_info=True)
time.sleep(1)
continue
while True:
log.info('waiting on data queue')
try:
data = data_queue.get(timeout=1)
except Queue.Empty:
try:
connection.process_data_events()
except AMQPConnectionError:
break
continue
log.info('got data to submit')
try:
channel.basic_publish(exchange='',
routing_key='sandbox',
body=data,
properties=properties,
mandatory=True)
except Exception:
log.error('submission failed', exc_info=True)
data_queue.put(data)
break
log.info('submitted data to broker')
def test_blocking_send_get():
connection = BlockingConnection(support.PARAMETERS)
# Open the channel
channel = connection.channel()
# Declare the queue
queue_name = support.tools.test_queue_name('blocking_send_get')
channel.queue_declare(queue=queue_name,
durable=False,
exclusive=True,
auto_delete=True)
message = ('test_blocking_send:%.4f' % time()).encode('utf-8')
channel.basic_publish(routing_key=queue_name,
exchange="",
body=message,
properties=BasicProperties(
content_type="text/plain",
delivery_mode=1))
# Loop while we try to get the message we sent
message_in = channel.basic_get(queue=queue_name)
# Close the connection
connection.close()
# Only check the body
if message_in[2] != message:
assert False, "Did not receive the same message back"
def blocking_submitter(parameters, data_queue,
properties=DEFAULT_PROPERTIES):
while True:
try:
connection = BlockingConnection(parameters)
channel = connection.channel()
channel.queue_declare(queue='sandbox', durable=True)
except Exception:
log.error('connection failure', exc_info=True)
time.sleep(1)
continue
while True:
log.info('waiting on data queue')
try:
data = data_queue.get(timeout=1)
except Queue.Empty:
try:
connection.process_data_events()
except AMQPConnectionError:
break
continue
log.info('got data to submit')
try:
channel.basic_publish(exchange='',
routing_key='sandbox',
body=data,
properties=properties,
mandatory=True)
except Exception:
log.error('submission failed', exc_info=True)
data_queue.put(data)
break
log.info('submitted data to broker')
def test_blocking_invalid_exchange():
# Connect to RabbitMQ
connection = BlockingConnection(support.PARAMETERS)
# Open the channel
channel = connection.channel()
# Declare the queue
queue_name = support.tools.test_queue_name('blocking_send_get')
channel.queue_declare(queue=queue_name,
durable=False,
exclusive=True,
auto_delete=True)
message = 'test_blocking_send:%.4f' % time()
try:
channel.basic_publish(exchange="invalid-exchange",
routing_key=queue_name,
body=message,
mandatory=True,
properties=BasicProperties(
content_type="text/plain", delivery_mode=1))
while True:
channel.transport.connection.process_data_events()
except AMQPChannelError, err:
if err[0] != 404:
assert False, "Did not receive a Channel.Close"
class RabbitMQProcessor(CrawlQueue):
"""class created to encapsulate the rabbit reading and rewriting proccess
to make common interface towards already existing crawle code
Author: Maximiliano Mendez
"""
def __init__(self, host, queue_name):
super(RabbitMQProcessor, self).__init__()
self.queue_name = queue_name
self.parameters = pika.ConnectionParameters(host)
self.connection = BlockingConnection(self.parameters)
self.channel = self.connection.channel()
self.channel.queue_declare(queue=self.queue_name, durable=True, exclusive=False, auto_delete=False)
def _get(self):
method, header, body = self.channel.basic_get(queue=self.queue_name)
if method.NAME == "Basic.GetEmpty":
raise Queue.Empty
req_res = pickle.loads(body)
self.channel.basic_ack(delivery_tag=method.delivery_tag)
return req_res
def _put(self, req_res):
message = pickle.dumps(req_res)
self.channel.basic_publish(
exchange="",
routing_key=self.queue_name,
body=message,
properties=pika.BasicProperties(content_type="text/plain", delivery_mode=1),
)
def stop(self):
self.connection.close()
def main(argv=None):
"""Main logic hit by the commandline invocation."""
parser = argparse.ArgumentParser(__doc__)
parser.add_argument('config', help="path to the configuration file")
args = parser.parse_args(argv)
if args.config is not None:
fileConfig(args.config)
logger.info("Logging initialized")
config = ConfigParser()
config.read(args.config)
# Grab the database uri setting from the config.
Session = create_database_session(config.get('anomaly', 'database-uri'))
# Queue initialization
connection = BlockingConnection()
channel = connection.channel()
# Declare the exchange and an unnamed queue.
channel.exchange_declare(exchange=EXCHANGE, type='topic')
declared_queue = channel.queue_declare(queue=QUEUE, durable=True,
exclusive=False)
channel.queue_bind(exchange=EXCHANGE, queue=QUEUE,
routing_key=BINDING_KEY)
# Setup up our consumer callback
channel.basic_consume(consumer, queue=QUEUE)
try:
channel.start_consuming()
except KeyboardInterrupt:
channel.stop_consuming()
connection.close()
def test_blocking_send_get():
connection = BlockingConnection(support.PARAMETERS)
# Open the channel
channel = connection.channel()
# Declare the queue
queue_name = support.tools.test_queue_name('blocking_send_get')
channel.queue_declare(queue=queue_name,
durable=False,
exclusive=True,
auto_delete=True)
message = 'test_blocking_send:%.4f' % time()
channel.basic_publish(routing_key=queue_name,
exchange="",
body=message,
properties=BasicProperties(content_type="text/plain",
delivery_mode=1))
# Loop while we try to get the message we sent
message_in = channel.basic_get(queue=queue_name)
# Close the connection
connection.close()
# Only check the body
if message_in[2] != message:
assert False, "Did not receive the same message back"
def test_blocking_send_get():
parameters = pika.ConnectionParameters(host=HOST, port=PORT)
connection = BlockingConnection(parameters)
# Open the channel
channel = connection.channel()
# Declare the queue
queue_name = support.tools.test_queue_name('blocking_send_get')
channel.queue_declare(queue=queue_name,
durable=False,
exclusive=True,
auto_delete=True)
message = 'test_blocking_send:%.4f' % time.time()
try:
channel.basic_publish(exchange='undeclared-exchange',
routing_key=queue_name,
body=message,
properties=pika.BasicProperties(
content_type="text/plain",
delivery_mode=1))
except pika.exceptions.AMQPChannelError, error:
if error[0] != 404:
assert False, "Did not receive a Channel.Close"
def test_blocking_invalid_exchange():
# Connect to RabbitMQ
connection = BlockingConnection(support.PARAMETERS)
# Open the channel
channel = connection.channel()
# Declare the queue
queue_name = support.tools.test_queue_name('blocking_send_get')
channel.queue_declare(queue=queue_name,
durable=False,
exclusive=True,
auto_delete=True)
message = 'test_blocking_send:%.4f' % time()
try:
channel.basic_publish(exchange="invalid-exchange",
routing_key=queue_name,
body=message,
mandatory=True,
properties=BasicProperties(
content_type="text/plain",
delivery_mode=1))
while True:
channel.transport.connection.process_data_events()
except AMQPChannelError, err:
if err[0] != 404:
assert False, "Did not receive a Channel.Close"
def test_blocking_send_get():
parameters = pika.ConnectionParameters(host=HOST, port=PORT)
connection = BlockingConnection(parameters)
# Open the channel
channel = connection.channel()
# Declare the queue
queue_name = support.tools.test_queue_name('blocking_send_get')
channel.queue_declare(queue=queue_name,
durable=False,
exclusive=True,
auto_delete=True)
message = 'test_blocking_send:%.4f' % time.time()
channel.basic_publish(exchange='',
routing_key=queue_name,
body=message,
properties=pika.BasicProperties(
content_type="text/plain",
delivery_mode=1))
# Loop while we try to get the message we sent
message_in = channel.basic_get(queue=queue_name)
# Close the connection
connection.close()
# Only check the body
if message_in[2] != message:
assert False, "Did not receive the same message back"
def main():
queue_name = "puccini_clean"
host = 'localhost'
conn = BlockingConnection(ConnectionParameters('localhost'))
channel = conn.channel()
channel.queue_declare(queue_name)
channel.basic_publish(exchange='',
routing_key=queue_name,
body=json.dumps({'clean': True}))
channel.close()
conn.close()
def main():
queue_name = "grq_clean"
host = 'localhost'
conn = BlockingConnection(ConnectionParameters('localhost'))
channel = conn.channel()
channel.queue_declare(queue_name, durable=True)
channel.basic_publish(exchange='',
routing_key=queue_name,
body=json.dumps({'clean': True}),
properties=BasicProperties(delivery_mode=2))
channel.close()
conn.close()
def start(db, messaging):
global connection
connection = BlockingConnection(pika.URLParameters(messaging))
global channel
channel = connection.channel()
channel.queue_declare(queue=EXCHANGE, durable=True, exclusive=False, auto_delete=False)
channel.exchange_declare(exchange=EXCHANGE)
channel.queue_bind(exchange=EXCHANGE, queue=EXCHANGE)
global database
client = MongoClient(db)
database = client.get_default_database()
app.run(host="0.0.0.0")
def main(argv=None):
"""Main logic hit by the commandline invocation."""
connection = BlockingConnection()
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True, exclusive=False)
# Convert the data over to JSON message bits relative to what the
# producer would send.
data = [json.dumps(dict(zip(XXX_DATA_STRUCT, d))) for d in XXX_DATA]
for message in data:
# Send the message to the preprocessor/incoming queue.
properties = BasicProperties(content_type="application/json",
delivery_mode=1)
channel.basic_publish(exchange='', routing_key=QUEUE, body=message,
properties=properties)
def main(argv=None):
"""Main logic hit by the commandline invocation."""
parser = argparse.ArgumentParser(__doc__)
parser.add_argument("config", help="path to the configuration file")
parser.add_argument("-n", "--name", help="drone name (used in configuration)")
args = parser.parse_args(argv)
if args.config is not None:
fileConfig(args.config)
logger.info("Logging initialized")
config = ConfigParser()
config.read(args.config)
# Retrieve the drone's settings from a generic section or one
# specified in the arguments.
config_section = "drone"
if args.name is not None:
config_section = "drone:{0}".format(args.name)
drone_settings = dict(config.items(config_section))
# XXX Used to initialize a sql session, but this should never
# happen because drones shouldn't have access to the
# persistent storage.
Session = create_database_session(config.get("anomaly", "database-uri"))
# Queue initialization
connection = BlockingConnection()
channel = connection.channel()
# Declare the exchange and an unnamed queue.
channel.exchange_declare(exchange=EXCHANGE, type="topic")
declared_queue = channel.queue_declare(exclusive=True)
queue_name = declared_queue.method.queue
channel.queue_bind(exchange=EXCHANGE, queue=queue_name, routing_key=drone_settings["binding-key"])
# Import the consumer from the settings line.
module_path, consumer_name = drone_settings["consumer-class"].split(":")
consumer_import = __import__(module_path, globals(), locals(), [consumer_name])
consumer_cls = getattr(consumer_import, consumer_name)
consumer = consumer_cls(drone_settings)
# Setup up our consumer callback
channel.basic_consume(consumer, queue=queue_name)
try:
channel.start_consuming()
except KeyboardInterrupt:
channel.stop_consuming()
connection.close()
def __init__(self, routing_key):
self.connection = BlockingConnection(
ConnectionParameters(host='33.33.33.10')
)
self.channel = self.connection.channel()
self.queue = self.channel.queue_declare(queue='test_metranome', exclusive=True, auto_delete=False)
self.channel.queue_bind(exchange='metranome', queue='test_metranome', routing_key=routing_key)
class RabbitMQProcessor(CrawlQueue):
"""class created to encapsulate the rabbit reading and rewriting proccess
to make common interface towards already existing crawle code
Author: Maximiliano Mendez
"""
def __init__(self, host, queue_name):
super(RabbitMQProcessor, self).__init__()
conn_retries = 4
connected = False
self.queue_name = queue_name
while conn_retries > 0 or not connected:
try:
self.parameters = pika.ConnectionParameters(host)
self.connection = BlockingConnection(self.parameters)
self.channel = self.connection.channel()
self.channel.queue_declare(queue=self.queue_name, durable=True,
exclusive=False, auto_delete=False)
connected = True
except Exception, e:
conn_retries -= 1
if not connected:
global STOP_CRAWLE
STOP_CRAWLE = True
self.stop()
def emit(self, record):
msg = self.format(record)
body = json.dumps(
{"msg": msg, "loggername": record.name, "level": record.levelname, "created": datetime.now().isoformat()}
)
try:
con = BlockingConnection(ConnectionParameters(self.host))
except socket.error:
raise RabbitConnectionException("Connection to {0} failed".format(self.host))
channel = con.channel()
channel.queue_declare(queue=self.queue, durable=True, exclusive=False, auto_delete=False)
channel.basic_publish(exchange="", routing_key=self.queue, body=body)
con.close()
def __init__(self, host, queue_name):
super(RabbitMQProcessor, self).__init__()
self.queue_name = queue_name
self.parameters = pika.ConnectionParameters(host)
self.connection = BlockingConnection(self.parameters)
self.channel = self.connection.channel()
self.channel.queue_declare(queue=self.queue_name, durable=True, exclusive=False, auto_delete=False)
def __init__(self):
self.exchange_name = "bu_outputs"
self.queue = "glue_script"
parameters = pika.ConnectionParameters('localhost')
self.connection = BlockingConnection(parameters) # Open conn to RabbitMQ with default params for localhost
self.channel = self.connection.channel() # Open the channel
self.channel.exchange_declare(exchange=self.exchange_name, type='direct', passive=False)
self.channel.queue_declare(queue=self.queue, durable=True,exclusive=False, auto_delete=False) # Declare a queue
self.channel.queue_bind(queue=self.queue, exchange=self.exchange_name, routing_key=self.queue)
def start(db, messaging):
global connection
connection = BlockingConnection(pika.URLParameters(messaging))
global channel
channel = connection.channel()
channel.queue_declare(queue=EXCHANGE,
durable=True,
exclusive=False,
auto_delete=False)
channel.exchange_declare(exchange=EXCHANGE)
channel.queue_bind(exchange=EXCHANGE, queue=EXCHANGE)
global database
client = MongoClient(db)
database = client.get_default_database()
app.run(host="0.0.0.0")
def connect(self, connection_params):
"""
Performs the actual connection to the RabbitMQ server.
:param connection_params: The connection parameters. See pika.ConnectionParameters for available properties
:type connection_params: dict
"""
connect_params_obj = ConnectionParameters(**connection_params)
self.connection = BlockingConnection(connect_params_obj)
self.channel = self.connection.channel()
def __init__(self, host, queue):
self.queue = queue
self.parameters = pika.ConnectionParameters(host)
pika.log.info("Establishing connection")
self.connection = BlockingConnection(self.parameters)
pika.log.info("About to declare queue")
pika.log.info("Queue declared")
def main(argv=None):
"""Main logic hit by the commandline invocation."""
logger.setLevel(logging.INFO)
connection = BlockingConnection()
channel = connection.channel()
# Declare the exchange and an unnamed queue.
channel.exchange_declare(exchange=EXCHANGE, type='topic')
declared_queue = channel.queue_declare(exclusive=True)
queue_name = declared_queue.method.queue
channel.queue_bind(exchange=EXCHANGE, queue=queue_name,
routing_key=BINDING_KEY)
# Setup up our consumer callback
channel.basic_consume(consumer, queue=queue_name)
try:
channel.start_consuming()
except KeyboardInterrupt:
channel.stop_consuming()
connection.close()
class Consumer(object):
def __init__(self, routing_key):
self.connection = BlockingConnection(
ConnectionParameters(host='33.33.33.10')
)
self.channel = self.connection.channel()
self.queue = self.channel.queue_declare(queue='test_metranome', exclusive=True, auto_delete=False)
self.channel.queue_bind(exchange='metranome', queue='test_metranome', routing_key=routing_key)
def get(self):
return self.channel.basic_get(queue='test_metranome', no_ack=True)
def _init_topic_conn(self):
self.connection = SelectConnection(self.conn_params, self.on_connected)
try:
# Loop so we can communicate with RabbitMQ
self.connection.ioloop.start()
except KeyboardInterrupt:
# Gracefully close the connection
self.connection.close()
# Loop until we're fully closed, will stop on its own
self.connection.ioloop.start()
else:
if DEBUG: print "Connection attempt rejected. Goodbye."
def initQueue(self):
# Connect to RabbitMQ
try:
parameters = pika.ConnectionParameters(**self.amqp)
connection = BlockingConnection(parameters)
# Open the channel
channel = connection.channel()
# Declare the queue
channel.queue_declare(queue=self.queue,
durable=False,
exclusive=False,
auto_delete=True)
print 'Sending notifications to queue: %s' % self.queue
except Exception:
print 'No notifications queue could be activated'
traceback.print_tb()
channel = connection = None
return (channel, connection)
def __init__(self, task_type):
self.task_type = task_type
cfg = config['manager']['message_queue']
credentials = PlainCredentials(cfg['username'], cfg['password'])
parameters = ConnectionParameters(
host=cfg['host'],
port=cfg['port'],
virtual_host=cfg['virtual_host'],
credentials=credentials)
conn = BlockingConnection(parameters)
self.channel = conn.channel()
self.exchange = '{0}.{1}'.format(
cfg['exchange'],
self.task_type)
log.info("declare_exchange")
self.channel.exchange_declare(
exchange=self.exchange,
type="fanout",
passive=False,
durable=False,
auto_delete=False)
def __call__(self, consuming_channel, method, header, body):
if header.content_type != 'application/json':
raise Exception("unrecognized message content-type: "
"{0}".format(header.content_type))
data = json.loads(body)
# Create the SQL Job entry and commit it.
type = data.get('type', '')
persistent_job = PersistentJob(body, type)
self.session.add(persistent_job)
self.session.commit()
id = persistent_job.id
# Create the new Job message object.
job = Job(id, data)
job.stamp()
logger.info("Created job {0}".format(job))
# Submit the new message to the topic exchange.
connection = BlockingConnection()
channel = connection.channel()
channel.exchange_declare(exchange=EXCHANGE, type='topic')
routing_key = get_routing_key(job)
message = jsonpickle.encode(job)
properties = BasicProperties(content_type="application/json")
channel.basic_publish(exchange=EXCHANGE,
routing_key=routing_key,
properties=properties,
body=message)
logger.debug("Sent message to '{0}' with {1!r}".format(routing_key,
message))
connection.close()
# Acknowledge message receipt
consuming_channel.basic_ack(method.delivery_tag)
class BlockingReader(object):
def __init__(self, host, queue):
self.queue = queue
self.parameters = pika.ConnectionParameters(host)
pika.log.info("Establishing connection")
self.connection = BlockingConnection(self.parameters)
pika.log.info("About to declare queue")
pika.log.info("Queue declared")
def read(self, channel):
pika.log.info("Reading single message")
method, header, body = channel.basic_get(queue=self.queue)
pika.log.info("Message received!")
channel.basic_ack(delivery_tag=method.delivery_tag)
def create_channel(self):
channel = self.connection.channel()
return channel
def stop(self):
self.connection.close()
def get_connection(self, timeout=None):
"""Connects to RabbitMQ and returns the connection object
Third Party (pika) Bug: https://github.com/pika/pika/issues/354 - Once
this bug is fixed we can take out our own retrying logic and use pika's
retry logic. In the mean time, connection failure messages will be
inaccurate; they'll say that only one connection attempt was made.
"""
return BlockingConnection(
ConnectionParameters(host=self.host,
credentials=self.get_credentials(),
connection_attempts=1,
retry_delay=0,
socket_timeout=timeout))
def __init__(self, host, queue_name):
super(RabbitMQProcessor, self).__init__()
conn_retries = 4
connected = False
self.queue_name = queue_name
while conn_retries > 0 or not connected:
try:
self.parameters = pika.ConnectionParameters(host)
self.connection = BlockingConnection(self.parameters)
self.channel = self.connection.channel()
self.channel.queue_declare(queue=self.queue_name, durable=True,
exclusive=False, auto_delete=False)
connected = True
except Exception, e:
conn_retries -= 1
class PikaDilly:
def __init__(self):
self.exchange_name = "bu_outputs"
self.queue = "glue_script"
parameters = pika.ConnectionParameters('localhost')
self.connection = BlockingConnection(parameters) # Open conn to RabbitMQ with default params for localhost
self.channel = self.connection.channel() # Open the channel
self.channel.exchange_declare(exchange=self.exchange_name, type='direct', passive=False)
self.channel.queue_declare(queue=self.queue, durable=True,exclusive=False, auto_delete=False) # Declare a queue
self.channel.queue_bind(queue=self.queue, exchange=self.exchange_name, routing_key=self.queue)
def publisher(self, _file):
# Send a message
try:
m = magic.Magic(mime=True)
mtype = str(m.from_file(_file))
self.channel.basic_publish(exchange=self.exchange_name, routing_key=self.queue, body=_file, properties=BasicProperties(content_type=mtype,delivery_mode=2))
except IOError, err:
pass
def connect(self):
host = self.host
if self.libc_gethostbyname: host = self.libc_gethostbyname(self.host)
while True:
if self.link and self.link.is_open:
try: self.link.close()
except: pass
try:
self.log.debug('Connecting to AMQP broker ({})'.format(host))
self.link = BlockingConnection(ConnectionParameters( host,
heartbeat=self.heartbeat, credentials=PlainCredentials(*self.auth) ))
# Even with BlockingConnection adapter,
# pika doesn't raise errors, unless you set callbacks to do that
self.link.set_backpressure_multiplier(2)
self.link.add_backpressure_callback(
ft.partial(self._error_callback, 'timeout') )
self.link.add_on_close_callback(
ft.partial(self._error_callback, 'closed/error') )
self.ch = self.link.channel()
self.schema_init()
if self.tx: self.ch.tx_select() # forces flush
except (self.PikaError, socket.error) as err:
self.log.exception('Connection to AMQP broker has failed: {}'.format(err))
delay = self.reconnect_info and self.reconnect_info[0] # first delay is 0
if delay:
self.log.debug('Will retry connection in {}s'.format(delay))
sleep(delay)
self.reconnect_info = self.reconnect_info or self.reconnect_delays
if len(self.reconnect_info) > 1: self.reconnect_info = self.reconnect_info[1:]
else:
self.reconnect_info = None
break
JSON_VALUE = '{"json_encoded": true, "value": "here", "random": %i}'
XML_VALUE = (
'<?xml version="1.0"><document><node><item>True</item><other attr'
'="foo">Bar</other><value>%i</value></node></document>'
)
YAML_VALUE = """%%YAML 1.2
---
Application:
poll_interval: 10.0
log_stats: True
name: Example
value: %i
"""
if __name__ == "__main__":
connection = BlockingConnection(ConnectionParameters())
# Open the channel
channel = connection.channel()
channel.exchange_declare(exchange="example")
# Declare the queue
channel.queue_declare(queue="generated_messages", durable=True, exclusive=False, auto_delete=False)
channel.queue_bind(exchange="example", queue="generated_messages", routing_key="rejected_example")
channel.queue_declare(queue="consumer_replies", durable=True, exclusive=False, auto_delete=False)
channel.queue_bind(exchange="example", queue="consumer_replies", routing_key="rejected_reply")
def test_blocking_consume():
parameters = pika.ConnectionParameters(host=HOST, port=PORT)
connection = BlockingConnection(parameters)
# Open the channel
channel = connection.channel()
# Declare the exchange
exchange_name = support.tools.test_queue_name('blocking_exchange')
frame = channel.exchange_declare(exchange=exchange_name,
type="direct",
auto_delete="true")
if not isinstance(frame.method, pika.spec.Exchange.DeclareOk):
assert False, \
"Did not receive Exchange.DeclareOk from channel.exchange_declare"
# Declare the queue
queue_name = support.tools.test_queue_name('blocking_consume')
frame = channel.queue_declare(queue=queue_name,
durable=False,
exclusive=True,
auto_delete=True)
if not isinstance(frame.method, pika.spec.Queue.DeclareOk):
assert False, \
"Did not receive Queue.DeclareOk from channel.queue_declare"
routing_key = "%s.%s" % (exchange_name, queue_name)
frame = channel.queue_bind(queue=queue_name,
exchange=exchange_name,
routing_key=routing_key)
if not isinstance(frame.method, pika.spec.Queue.BindOk):
assert False, \
"Did not receive Queue.BindOk from channel.queue_bind"
_sent = []
_received = []
@pika.log.method_call
def _on_message(channel, method, header, body):
_received.append(body)
if len(_received) == MESSAGES:
channel.stop_consuming()
if start < time.time() - 2:
assert False, "Test timed out"
for x in xrange(0, MESSAGES):
message = 'test_blocking_send:%i:%.4f' % (x, time.time())
_sent.append(message)
channel.basic_publish(exchange=exchange_name,
routing_key=routing_key,
body=message,
properties=pika.BasicProperties(
content_type="text/plain",
delivery_mode=1))
# Loop while we get messages (for 2 seconds)
start = time.time()
# This is blocking
channel.basic_consume(consumer=_on_message, queue=queue_name, no_ack=True)
connection.close()
# Check our results
if len(_sent) != MESSAGES:
assert False, "We did not send the expected qty of messages: %i" %\
len(_sent)
if len(_received) != MESSAGES:
assert False, "Did not receive the expected qty of messages: %i" %\
len(_received)
for message in _received:
if message not in _sent:
assert False, 'Received a message we did not send.'
for message in _sent:
if message not in _received:
assert False, 'Sent a message we did not receive.'
"""
Example of the use of basic_get. NOT RECOMMENDED for fast consuming - use
basic_consume instead if at all possible!
"""
import sys
import time
from pika.adapters import BlockingConnection
from pika.connection import ConnectionParameters
if __name__ == '__main__':
# Connect to RabbitMQ
host = (len(sys.argv) > 1) and sys.argv[1] or '127.0.0.1'
connection = BlockingConnection(ConnectionParameters(host))
# Open the channel
channel = connection.channel()
# Declare the queue
channel.queue_declare(queue="test",
durable=True,
exclusive=False,
auto_delete=False)
# Initialize our timers and loop until external influence stops us
while connection.is_open:
# Call basic get which returns the 3 frame types
method, header, body = channel.basic_get(queue="test")
def test_blocking_consume_clean_close():
# Connect to RabbitMQ
connection = BlockingConnection(support.PARAMETERS)
# Open the channel
channel = connection.channel()
# Declare the exchange
exchange_name = support.tools.test_queue_name('blocking_exchange')
frame = channel.exchange_declare(exchange=exchange_name,
type="direct",
auto_delete=True)
if not isinstance(frame.method, Exchange.DeclareOk):
assert False, \
"Did not receive Exchange.DeclareOk from channel.exchange_declare"
# Declare the queue
queue_name = support.tools.test_queue_name('blocking_consume')
frame = channel.queue_declare(queue=queue_name,
durable=False,
exclusive=True,
auto_delete=True)
if not isinstance(frame.method, Queue.DeclareOk):
assert False, \
"Did not receive Queue.DeclareOk from channel.queue_declare"
routing_key = "%s.%s" % (exchange_name, queue_name)
frame = channel.queue_bind(queue=queue_name,
exchange=exchange_name,
routing_key=routing_key)
if not isinstance(frame.method, Queue.BindOk):
assert False, \
"Did not receive Queue.BindOk from channel.queue_bind"
_sent = []
_received = []
def _on_message(channel, method, header, body):
_received.append(body)
if len(_received) == MESSAGES:
return connection.close()
if start < time() - MAX_DURATION:
assert False, "Test timed out"
print "Here"
for x in xrange(0, MESSAGES):
message = 'test_blocking_send:%i:%.4f' % (x, time())
_sent.append(message)
channel.basic_publish(exchange=exchange_name,
routing_key=routing_key,
body=message,
properties=BasicProperties(
content_type="text/plain", delivery_mode=1))
# Loop while we get messages (for 2 seconds)
start = time()
channel.basic_consume(consumer_callback=_on_message,
queue=queue_name,
no_ack=True)
# This is blocking
channel.start_consuming()
# Check our results
if len(_sent) != MESSAGES:
assert False, "We did not send the expected qty of messages: %i" %\
len(_sent)
if len(_received) != MESSAGES:
assert False, "Did not receive the expected qty of messages: %i" %\
len(_received)
for message in _received:
if message not in _sent:
assert False, 'Received a message we did not send.'
for message in _sent:
if message not in _received:
assert False, 'Sent a message we did not receive.'
#!/usr/bin/env python
from pika.adapters import BlockingConnection
from pika import BasicProperties
connection = BlockingConnection()
channel = connection.channel()
queue_name = 'microservice.queue.1'
args = {"x-queue-master-locator": "min-masters"}
channel.queue_declare(queue=queue_name, durable=True, arguments=args)
channel.basic_publish(exchange='', routing_key=queue_name, body='Test message')
print(" [x] Sent 'test message!'")
connection.close()
def __init__(self):
super(QueueThread, self).__init__()
self.name = self.__class__.__name__
self.connection = BlockingConnection()
self._stop = threading.Event()
self.setDaemon(True)
stdOut = pop.stdout.read().decode()
stdErr = pop.stderr.read().decode()
for line in stdOut.split('\n'):
if line.startswith(QUEUE_NAME):
COUNT = int(line.split()[1])
break
print(("Total number of messages in %s: %d" % (QUEUE_NAME, COUNT)))
if COUNT == 0:
sys.exit()
# Connect to RabbitMQ
match = DEF_HOST.search(app.conf['BROKER_URL'])
broker_url = "{}%2F".format(
match.group(1)) if match else app.conf['BROKER_URL']
print("broker_url: {}".format(broker_url))
CONNECTION = BlockingConnection(URLParameters(broker_url))
# Open the channel
channel = CONNECTION.channel()
# Declare the queue
channel.queue_declare(queue=QUEUE_NAME,
durable=True,
exclusive=False,
auto_delete=False,
arguments={'x-max-priority': 10})
# Add a queue to consume
channel.basic_consume(handle_delivery, queue=QUEUE_NAME)
# Start consuming, block until keyboard interrupt
class RabbitObj(object):
'''
This is the class inherited by all other objects that work with RabbitMQ
within CUFF.
'''
def __init__(self,
host='localhost',
port=spec.PORT,
virtual_host='/',
credentials=None,
channel_max=0,
frame_max=spec.FRAME_MAX_SIZE,
heartbeat=False):
'''
Initializes all the connection parameters and stores them in a
ConnectionParameters object at self.conn_params.
Parameters:
- host: Hostname or IP Address to connect to, defaults to localhost.
- port: TCP port to connect to, defaults to 5672
- virtual_host: RabbitMQ virtual host to use, defaults to /
- credentials: A instance of a credentials class to authenticate with.
Defaults to PlainCredentials for the guest user.
- channel_max: Maximum number of channels to allow, defaults to 0 for
None
- frame_max: The maximum byte size for an AMQP frame. Defaults to
131072
- heartbeat: Turn heartbeat checking on or off. Defaults to False.
'''
# Send the values to a ConnectionParameters object, which has built-in
# type-checking
if isinstance(host, unicode): host = str(host)
self.conn_params = pika.ConnectionParameters(host, port, virtual_host,
credentials, channel_max,
frame_max, heartbeat)
self.connection = None
self.channel = None
self.init_callback = None
self.exchange = None
self.routing_key = None
self.queue_created = False
self.queue_name = None
self.no_ack = False
self._debug_prefix = ""
# The binding key is essentially the same thing as a routing key, only it is
# used by receivers instead of senders. We set it to None here so that we can
# detect on the fly if we have executed a queue_bind() for the binding key
# before we attempt to receive messages. This is also how we prevent senders
# from performing a queue_bind(), since they don't generally do that.
self.binding_key = None
def init_connection(self, callback, queue_name='', exchange='',
exchange_type='topic', routing_key="#", blocking=False,
user_id=None):
'''
Handles all connection, channel, etc. issues involved in fully
connecting to the RabbitMQ server. The function 'callback' is stored
and called once all steps have successfully completed. Using this
method forces all communication to go through the queue specified by
'queue'.
'''
self.exchange = str(exchange)
self.init_callback = callback
if user_id != None:
self.user_id = str(user_id)
if exchange_type not in EXCHANGE_TYPES:
raise ValueError("Exchange type must be one of: %s" % \
str(EXCHANGE_TYPES))
else:
self.ex_type = str(exchange_type)
if isinstance(queue_name, unicode):
queue_name = str(queue_name)
elif not isinstance(queue_name, str):
raise TypeError("Queue must be a str, got %s instead" % \
str(type(queue_name)))
self.queue_name = queue_name
if isinstance(routing_key, unicode):
routing_key = str(routing_key)
elif not isinstance(routing_key, str):
raise TypeError("Routing key must be a str, got %s instead" % \
str(type(queue_name)))
self.routing_key = routing_key
if DEBUG: print self._debug_prefix + "About to start connection...",
# From here, determine how to initialize the connection further by
# what type of exchange was requested.
if self.ex_type == "topic":
self._init_topic_conn()
#elif self.ex_type == "direct":
# self._init_direct_conn()
elif not blocking:
self._init_topic_conn()
else:
raise NotImplementedError("Only 'topic' and 'direct' exchange " \
"types are currently supported.")
def _init_topic_conn(self):
self.connection = SelectConnection(self.conn_params, self.on_connected)
try:
# Loop so we can communicate with RabbitMQ
self.connection.ioloop.start()
except KeyboardInterrupt:
# Gracefully close the connection
self.connection.close()
# Loop until we're fully closed, will stop on its own
self.connection.ioloop.start()
else:
if DEBUG: print "Connection attempt rejected. Goodbye."
def _init_direct_conn(self):
blocking = True
self.connection = BlockingConnection(self.conn_params)
self.on_connected(self.connection, blocking)
self.channel = self.connection.channel()
self.on_channel_open(self.channel)
def on_connected(self, connection, blocking=False):
'''
'''
if DEBUG:
print self._debug_prefix + "Connected\n Host: %s\n " \
"Exchange: %s\n" % (self.conn_params.host, self.exchange) + \
self._debug_prefix + "Creating channel...",
# These should always be the same, but just in case...
if self.connection is not connection:
# Adopt the new connection object
self.connection = connection
if not blocking:
self.connection.channel(on_open_callback=self.on_channel_open)
def on_channel_open(self, new_channel):
'''
'''
if DEBUG:
print "Created\n" + self._debug_prefix + \
"Declaring %s exchange: '%s'" % (self.ex_type, self.exchange)
self.channel = new_channel
self.channel.exchange_declare(exchange=self.exchange,
durable=True,
type=self.ex_type,
callback=self.init_callback)
#TODO: Mike, I added some correlation_id stuff here.
# Using this TODO as a marker so you can find it
def send_message(self, body, routing_key=None, correlation_id=None):
'''
Sends a message to the exchange on the server to which a connection
has already been established. Both parameters 'body' and 'routing_key'
must be of type str, or a TypeError will be raised. If no routing_key
is specified, it defaults to the routing key value specified at
declaration.
'''
if type(body) != str:
raise TypeError("Parameter 'body' must be of type 'str', got " \
"'%s' instead." % type(body))
if routing_key == None:
routing_key = self.routing_key
if type(routing_key) != str:
raise TypeError("Parameter 'routing_key' must be of type 'str', " \
"got '%s' instead." % type(routing_key))
#if INFO: print "\n" + self._debug_prefix + "Sending message on %s : %s : %s" % (self.exchange, self.queue_name, routing_key)
# TODO: In the following, create a means of catching "unroutable" messages (necessary because the 'mandatory'
# flag is set)
props = pika.BasicProperties(delivery_mode=2, # Persistent messages
user_id=str(self.user_id))
if DEBUG:
print "[x] Sending message to %s" % (routing_key)
self.channel.basic_publish(exchange=self.exchange,
routing_key=routing_key,
body=body,
mandatory=True,
properties=props,
)
def create_queue(self, durable=True, exclusive=False, name=None,
callback=None):
'''
Typically don't change the default parameters.
'''
if not name:
name = self.queue_name
if DEBUG: print self._debug_prefix + "Creating queue '%s'..." % name
self.queue_created = True
self.channel.queue_declare(durable=durable, exclusive=exclusive,
queue=name, callback=callback)
def bind_routing_key(self, frame):
if DEBUG:
print self._debug_prefix + "Binding queue '%s' to key '%s'" % \
(self.queue_name, self.routing_key)
self.binding_key = self.routing_key # Signals that we've performed the queue binding
self.channel.queue_bind(exchange=self.exchange, queue=self.queue_name,
routing_key=self.binding_key,
callback=self._rcv())
def receive_message(self, callback=None, no_ack=False, frame=None):
'''
Retrieves a message from the channel and queue specified in earlier
initialization code. The callback function defaults to
self.process_msg. Only under rare circumstances should no_ack be set
to True, since this will not inform RabbitMQ that completed tasks can
be removed from the queue.
'''
if DEBUG: print self._debug_prefix + "Preparing to receive"
# Set the callback function for processing received messages
if callback == None:
self.rcv_callback = self.process_msg
else:
self.rcv_callback = callback
self.no_ack = no_ack
# Check the queue, declare if necessary
if not self.queue_created:
self.create_queue(callback=self.bind_routing_key)
# Check the binding key, bind the channel if necessary
elif self.binding_key == None:
if frame == None:
raise TypeError("Parameter 'frame' must not be None if the " \
"routing key has not yet been bound.")
self.bind_routing_key(frame) # self._rcv() is specified as the callback in this method
else:
self._rcv()
def _rcv(self):
self.channel.basic_qos(prefetch_count=1)
if DEBUG: print self._debug_prefix + "Consuming..."
self.channel.basic_consume(consumer_callback=self.rcv_callback,
queue=self.queue_name, no_ack=self.no_ack)
def ack(self, tag):
self.channel.basic_ack(delivery_tag=tag)
def reject(self, tag):
self.channel.basic_reject(delivery_tag=tag)
def process_msg(self, channel, method, header, body):
raise NotImplementedError
env=os.environ)
try:
sts = pop.wait() #wait for child to terminate and get status
except Exception, e:
print str(e)
status = pop.returncode
#print "returncode is:",status
stdOut = pop.stdout.read()
stdErr = pop.stderr.read()
for line in stdOut.split('\n'):
if line.startswith("error_queue"):
COUNT = int(line.split()[1])
break
print "Total number of messages in error_queue:", COUNT
if COUNT == 0: sys.exit()
# Connect to RabbitMQ
host = (len(sys.argv) > 1) and sys.argv[1] or '127.0.0.1'
CONNECTION = BlockingConnection(ConnectionParameters(host))
# Open the channel
channel = CONNECTION.channel()
# loop
for method_frame, properties, body in channel.consume('error_queue'):
if handle_delivery(channel, method_frame, body): break
channel.stop_consuming()
channel.close()
CONNECTION.close()
HTML_VALUE = '<html><head><title>Hi</title></head><body>Hello %i</body></html>'
JSON_VALUE = '{"json_encoded": true, "value": "here", "random": %i}'
XML_VALUE = '<?xml version="1.0"><document><node><item>True</item><other attr' \
'="foo">Bar</other><value>%i</value></node></document>'
YAML_VALUE = """%%YAML 1.2
---
Application:
poll_interval: 10.0
log_stats: True
name: Example
value: %i
"""
if __name__ == '__main__':
connection = BlockingConnection(ConnectionParameters())
channel = connection.channel()
channel.exchange_declare(exchange='examples',
exchange_type='topic',
durable=True)
print('Declared the examples exchange')
channel.queue_declare(queue='sync_example',
durable=True,
exclusive=False,
auto_delete=False)
channel.queue_bind(exchange='examples',
queue='sync_example',
routing_key='example.sync')
print('Declared and bound sync_example')
def start(db, messaging):
connection = BlockingConnection(pika.URLParameters(messaging))
publish_connection = BlockingConnection(pika.URLParameters(messaging))
client = MongoClient(db)
global database
database = client.get_default_database()
channel = connection.channel()
global pub_channel
pub_channel = publish_connection.channel()
channel = connection.channel()
channel.queue_declare(queue=EXCHANGE,
durable=True,
exclusive=False,
auto_delete=False)
channel.exchange_declare(exchange=EXCHANGE)
channel.queue_bind(exchange=EXCHANGE, queue=EXCHANGE)
channel.basic_consume(on_message, EXCHANGE)
try:
channel.start_consuming()
except KeyboardInterrupt:
channel.stop_consuming()
connection.close()
publish_connection.close()
if __name__ == '__main__':
# Connect to RabbitMQ
host = 'spedata6'
port = 5672
vhost = '/'
credentials = pika.PlainCredentials('guest', 'guest')
parameters = pika.ConnectionParameters(host=host,
port=port,
virtual_host=vhost,
credentials=credentials,
channel_max=0,
frame_max=131072,
heartbeat=False)
connection = BlockingConnection(parameters)
# Open the channel
channel = connection.channel()
# Declare the exchange
channel.exchange_declare(exchange='rawdata',
type='fanout',
passive=False,
durable=True,
auto_delete=False,
internal=False,
nowait=False,
arguments={})
# Declare the queue
