def start(db, messaging): connection = BlockingConnection(pika.URLParameters(messaging)) publish_connection = BlockingConnection(pika.URLParameters(messaging)) client = MongoClient(db) global database database = client.get_default_database() channel = connection.channel() global pub_channel pub_channel = publish_connection.channel() channel = connection.channel() channel.queue_declare(queue=EXCHANGE, durable=True, exclusive=False, auto_delete=False) channel.exchange_declare(exchange=EXCHANGE) channel.queue_bind(exchange=EXCHANGE, queue=EXCHANGE) channel.basic_consume(on_message, EXCHANGE) try: channel.start_consuming() except KeyboardInterrupt: channel.stop_consuming() connection.close() publish_connection.close()
def test_blocking_send_get(): connection = BlockingConnection(support.PARAMETERS) # Open the channel channel = connection.channel() # Declare the queue queue_name = support.tools.test_queue_name('blocking_send_get') channel.queue_declare(queue=queue_name, durable=False, exclusive=True, auto_delete=True) message = 'test_blocking_send:%.4f' % time() channel.basic_publish(routing_key=queue_name, exchange="", body=message, properties=BasicProperties(content_type="text/plain", delivery_mode=1)) # Loop while we try to get the message we sent message_in = channel.basic_get(queue=queue_name) # Close the connection connection.close() # Only check the body if message_in[2] != message: assert False, "Did not receive the same message back"
def test_blocking_send_get(): parameters = pika.ConnectionParameters(host=HOST, port=PORT) connection = BlockingConnection(parameters) # Open the channel channel = connection.channel() # Declare the queue queue_name = support.tools.test_queue_name('blocking_send_get') channel.queue_declare(queue=queue_name, durable=False, exclusive=True, auto_delete=True) message = 'test_blocking_send:%.4f' % time.time() channel.basic_publish(exchange='', routing_key=queue_name, body=message, properties=pika.BasicProperties( content_type="text/plain", delivery_mode=1)) # Loop while we try to get the message we sent message_in = channel.basic_get(queue=queue_name) # Close the connection connection.close() # Only check the body if message_in[2] != message: assert False, "Did not receive the same message back"
def test_blocking_send_get(): parameters = pika.ConnectionParameters(host=HOST, port=PORT) connection = BlockingConnection(parameters) # Open the channel channel = connection.channel() # Declare the queue queue_name = support.tools.test_queue_name('blocking_send_get') channel.queue_declare(queue=queue_name, durable=False, exclusive=True, auto_delete=True) message = 'test_blocking_send:%.4f' % time.time() try: channel.basic_publish(exchange='undeclared-exchange', routing_key=queue_name, body=message, properties=pika.BasicProperties( content_type="text/plain", delivery_mode=1)) except pika.exceptions.AMQPChannelError, error: if error[0] != 404: assert False, "Did not receive a Channel.Close"
def test_blocking_send_get(): connection = BlockingConnection(support.PARAMETERS) # Open the channel channel = connection.channel() # Declare the queue queue_name = support.tools.test_queue_name('blocking_send_get') channel.queue_declare(queue=queue_name, durable=False, exclusive=True, auto_delete=True) message = ('test_blocking_send:%.4f' % time()).encode('utf-8') channel.basic_publish(routing_key=queue_name, exchange="", body=message, properties=BasicProperties( content_type="text/plain", delivery_mode=1)) # Loop while we try to get the message we sent message_in = channel.basic_get(queue=queue_name) # Close the connection connection.close() # Only check the body if message_in[2] != message: assert False, "Did not receive the same message back"
class RabbitMQProcessor(CrawlQueue): """class created to encapsulate the rabbit reading and rewriting proccess to make common interface towards already existing crawle code Author: Maximiliano Mendez """ def __init__(self, host, queue_name): super(RabbitMQProcessor, self).__init__() self.queue_name = queue_name self.parameters = pika.ConnectionParameters(host) self.connection = BlockingConnection(self.parameters) self.channel = self.connection.channel() self.channel.queue_declare(queue=self.queue_name, durable=True, exclusive=False, auto_delete=False) def _get(self): method, header, body = self.channel.basic_get(queue=self.queue_name) if method.NAME == "Basic.GetEmpty": raise Queue.Empty req_res = pickle.loads(body) self.channel.basic_ack(delivery_tag=method.delivery_tag) return req_res def _put(self, req_res): message = pickle.dumps(req_res) self.channel.basic_publish( exchange="", routing_key=self.queue_name, body=message, properties=pika.BasicProperties(content_type="text/plain", delivery_mode=1), ) def stop(self): self.connection.close()
class RabbitMQProcessor(CrawlQueue): """class created to encapsulate the rabbit reading and rewriting proccess to make common interface towards already existing crawle code Author: Maximiliano Mendez """ def __init__(self, host, queue_name): super(RabbitMQProcessor, self).__init__() conn_retries = 4 connected = False self.queue_name = queue_name while conn_retries > 0 or not connected: try: self.parameters = pika.ConnectionParameters(host) self.connection = BlockingConnection(self.parameters) self.channel = self.connection.channel() self.channel.queue_declare(queue=self.queue_name, durable=True, exclusive=False, auto_delete=False) connected = True except Exception, e: conn_retries -= 1 if not connected: global STOP_CRAWLE STOP_CRAWLE = True self.stop()
def main(argv=None): """Main logic hit by the commandline invocation.""" parser = argparse.ArgumentParser(__doc__) parser.add_argument('config', help="path to the configuration file") args = parser.parse_args(argv) if args.config is not None: fileConfig(args.config) logger.info("Logging initialized") config = ConfigParser() config.read(args.config) # Grab the database uri setting from the config. Session = create_database_session(config.get('anomaly', 'database-uri')) # Queue initialization connection = BlockingConnection() channel = connection.channel() # Declare the exchange and an unnamed queue. channel.exchange_declare(exchange=EXCHANGE, type='topic') declared_queue = channel.queue_declare(queue=QUEUE, durable=True, exclusive=False) channel.queue_bind(exchange=EXCHANGE, queue=QUEUE, routing_key=BINDING_KEY) # Setup up our consumer callback channel.basic_consume(consumer, queue=QUEUE) try: channel.start_consuming() except KeyboardInterrupt: channel.stop_consuming() connection.close()
def test_blocking_invalid_exchange(): # Connect to RabbitMQ connection = BlockingConnection(support.PARAMETERS) # Open the channel channel = connection.channel() # Declare the queue queue_name = support.tools.test_queue_name('blocking_send_get') channel.queue_declare(queue=queue_name, durable=False, exclusive=True, auto_delete=True) message = 'test_blocking_send:%.4f' % time() try: channel.basic_publish(exchange="invalid-exchange", routing_key=queue_name, body=message, mandatory=True, properties=BasicProperties( content_type="text/plain", delivery_mode=1)) while True: channel.transport.connection.process_data_events() except AMQPChannelError, err: if err[0] != 404: assert False, "Did not receive a Channel.Close"
def blocking_submitter(parameters, data_queue, properties=DEFAULT_PROPERTIES): while True: try: connection = BlockingConnection(parameters) channel = connection.channel() channel.queue_declare(queue='sandbox', durable=True) except Exception: log.error('connection failure', exc_info=True) time.sleep(1) continue while True: log.info('waiting on data queue') try: data = data_queue.get(timeout=1) except Queue.Empty: try: connection.process_data_events() except AMQPConnectionError: break continue log.info('got data to submit') try: channel.basic_publish(exchange='', routing_key='sandbox', body=data, properties=properties, mandatory=True) except Exception: log.error('submission failed', exc_info=True) data_queue.put(data) break log.info('submitted data to broker')
class Consumer(object): def __init__(self, routing_key): self.connection = BlockingConnection( ConnectionParameters(host='33.33.33.10') ) self.channel = self.connection.channel() self.queue = self.channel.queue_declare(queue='test_metranome', exclusive=True, auto_delete=False) self.channel.queue_bind(exchange='metranome', queue='test_metranome', routing_key=routing_key) def get(self): return self.channel.basic_get(queue='test_metranome', no_ack=True)
def main(): queue_name = "puccini_clean" host = 'localhost' conn = BlockingConnection(ConnectionParameters('localhost')) channel = conn.channel() channel.queue_declare(queue_name) channel.basic_publish(exchange='', routing_key=queue_name, body=json.dumps({'clean': True})) channel.close() conn.close()
def main(): queue_name = "grq_clean" host = 'localhost' conn = BlockingConnection(ConnectionParameters('localhost')) channel = conn.channel() channel.queue_declare(queue_name, durable=True) channel.basic_publish(exchange='', routing_key=queue_name, body=json.dumps({'clean': True}), properties=BasicProperties(delivery_mode=2)) channel.close() conn.close()
def main(argv=None): """Main logic hit by the commandline invocation.""" connection = BlockingConnection() channel = connection.channel() channel.queue_declare(queue=QUEUE, durable=True, exclusive=False) # Convert the data over to JSON message bits relative to what the # producer would send. data = [json.dumps(dict(zip(XXX_DATA_STRUCT, d))) for d in XXX_DATA] for message in data: # Send the message to the preprocessor/incoming queue. properties = BasicProperties(content_type="application/json", delivery_mode=1) channel.basic_publish(exchange='', routing_key=QUEUE, body=message, properties=properties)
def start(db, messaging): global connection connection = BlockingConnection(pika.URLParameters(messaging)) global channel channel = connection.channel() channel.queue_declare(queue=EXCHANGE, durable=True, exclusive=False, auto_delete=False) channel.exchange_declare(exchange=EXCHANGE) channel.queue_bind(exchange=EXCHANGE, queue=EXCHANGE) global database client = MongoClient(db) database = client.get_default_database() app.run(host="0.0.0.0")
def main(argv=None): """Main logic hit by the commandline invocation.""" parser = argparse.ArgumentParser(__doc__) parser.add_argument("config", help="path to the configuration file") parser.add_argument("-n", "--name", help="drone name (used in configuration)") args = parser.parse_args(argv) if args.config is not None: fileConfig(args.config) logger.info("Logging initialized") config = ConfigParser() config.read(args.config) # Retrieve the drone's settings from a generic section or one # specified in the arguments. config_section = "drone" if args.name is not None: config_section = "drone:{0}".format(args.name) drone_settings = dict(config.items(config_section)) # XXX Used to initialize a sql session, but this should never # happen because drones shouldn't have access to the # persistent storage. Session = create_database_session(config.get("anomaly", "database-uri")) # Queue initialization connection = BlockingConnection() channel = connection.channel() # Declare the exchange and an unnamed queue. channel.exchange_declare(exchange=EXCHANGE, type="topic") declared_queue = channel.queue_declare(exclusive=True) queue_name = declared_queue.method.queue channel.queue_bind(exchange=EXCHANGE, queue=queue_name, routing_key=drone_settings["binding-key"]) # Import the consumer from the settings line. module_path, consumer_name = drone_settings["consumer-class"].split(":") consumer_import = __import__(module_path, globals(), locals(), [consumer_name]) consumer_cls = getattr(consumer_import, consumer_name) consumer = consumer_cls(drone_settings) # Setup up our consumer callback channel.basic_consume(consumer, queue=queue_name) try: channel.start_consuming() except KeyboardInterrupt: channel.stop_consuming() connection.close()
def emit(self, record): msg = self.format(record) body = json.dumps( {"msg": msg, "loggername": record.name, "level": record.levelname, "created": datetime.now().isoformat()} ) try: con = BlockingConnection(ConnectionParameters(self.host)) except socket.error: raise RabbitConnectionException("Connection to {0} failed".format(self.host)) channel = con.channel() channel.queue_declare(queue=self.queue, durable=True, exclusive=False, auto_delete=False) channel.basic_publish(exchange="", routing_key=self.queue, body=body) con.close()
class PikaDilly: def __init__(self): self.exchange_name = "bu_outputs" self.queue = "glue_script" parameters = pika.ConnectionParameters('localhost') self.connection = BlockingConnection(parameters) # Open conn to RabbitMQ with default params for localhost self.channel = self.connection.channel() # Open the channel self.channel.exchange_declare(exchange=self.exchange_name, type='direct', passive=False) self.channel.queue_declare(queue=self.queue, durable=True,exclusive=False, auto_delete=False) # Declare a queue self.channel.queue_bind(queue=self.queue, exchange=self.exchange_name, routing_key=self.queue) def publisher(self, _file): # Send a message try: m = magic.Magic(mime=True) mtype = str(m.from_file(_file)) self.channel.basic_publish(exchange=self.exchange_name, routing_key=self.queue, body=_file, properties=BasicProperties(content_type=mtype,delivery_mode=2)) except IOError, err: pass
def main(argv=None): """Main logic hit by the commandline invocation.""" logger.setLevel(logging.INFO) connection = BlockingConnection() channel = connection.channel() # Declare the exchange and an unnamed queue. channel.exchange_declare(exchange=EXCHANGE, type='topic') declared_queue = channel.queue_declare(exclusive=True) queue_name = declared_queue.method.queue channel.queue_bind(exchange=EXCHANGE, queue=queue_name, routing_key=BINDING_KEY) # Setup up our consumer callback channel.basic_consume(consumer, queue=queue_name) try: channel.start_consuming() except KeyboardInterrupt: channel.stop_consuming() connection.close()
def initQueue(self): # Connect to RabbitMQ try: parameters = pika.ConnectionParameters(**self.amqp) connection = BlockingConnection(parameters) # Open the channel channel = connection.channel() # Declare the queue channel.queue_declare(queue=self.queue, durable=False, exclusive=False, auto_delete=True) print 'Sending notifications to queue: %s' % self.queue except Exception: print 'No notifications queue could be activated' traceback.print_tb() channel = connection = None return (channel, connection)
def __call__(self, consuming_channel, method, header, body): if header.content_type != 'application/json': raise Exception("unrecognized message content-type: " "{0}".format(header.content_type)) data = json.loads(body) # Create the SQL Job entry and commit it. type = data.get('type', '') persistent_job = PersistentJob(body, type) self.session.add(persistent_job) self.session.commit() id = persistent_job.id # Create the new Job message object. job = Job(id, data) job.stamp() logger.info("Created job {0}".format(job)) # Submit the new message to the topic exchange. connection = BlockingConnection() channel = connection.channel() channel.exchange_declare(exchange=EXCHANGE, type='topic') routing_key = get_routing_key(job) message = jsonpickle.encode(job) properties = BasicProperties(content_type="application/json") channel.basic_publish(exchange=EXCHANGE, routing_key=routing_key, properties=properties, body=message) logger.debug("Sent message to '{0}' with {1!r}".format(routing_key, message)) connection.close() # Acknowledge message receipt consuming_channel.basic_ack(method.delivery_tag)
class BlockingReader(object): def __init__(self, host, queue): self.queue = queue self.parameters = pika.ConnectionParameters(host) pika.log.info("Establishing connection") self.connection = BlockingConnection(self.parameters) pika.log.info("About to declare queue") pika.log.info("Queue declared") def read(self, channel): pika.log.info("Reading single message") method, header, body = channel.basic_get(queue=self.queue) pika.log.info("Message received!") channel.basic_ack(delivery_tag=method.delivery_tag) def create_channel(self): channel = self.connection.channel() return channel def stop(self): self.connection.close()
def __init__(self, task_type): self.task_type = task_type cfg = config['manager']['message_queue'] credentials = PlainCredentials(cfg['username'], cfg['password']) parameters = ConnectionParameters( host=cfg['host'], port=cfg['port'], virtual_host=cfg['virtual_host'], credentials=credentials) conn = BlockingConnection(parameters) self.channel = conn.channel() self.exchange = '{0}.{1}'.format( cfg['exchange'], self.task_type) log.info("declare_exchange") self.channel.exchange_declare( exchange=self.exchange, type="fanout", passive=False, durable=False, auto_delete=False)
def test_blocking_consume(): parameters = pika.ConnectionParameters(host=HOST, port=PORT) connection = BlockingConnection(parameters) # Open the channel channel = connection.channel() # Declare the exchange exchange_name = support.tools.test_queue_name('blocking_exchange') frame = channel.exchange_declare(exchange=exchange_name, type="direct", auto_delete="true") if not isinstance(frame.method, pika.spec.Exchange.DeclareOk): assert False, \ "Did not receive Exchange.DeclareOk from channel.exchange_declare" # Declare the queue queue_name = support.tools.test_queue_name('blocking_consume') frame = channel.queue_declare(queue=queue_name, durable=False, exclusive=True, auto_delete=True) if not isinstance(frame.method, pika.spec.Queue.DeclareOk): assert False, \ "Did not receive Queue.DeclareOk from channel.queue_declare" routing_key = "%s.%s" % (exchange_name, queue_name) frame = channel.queue_bind(queue=queue_name, exchange=exchange_name, routing_key=routing_key) if not isinstance(frame.method, pika.spec.Queue.BindOk): assert False, \ "Did not receive Queue.BindOk from channel.queue_bind" _sent = [] _received = [] @pika.log.method_call def _on_message(channel, method, header, body): _received.append(body) if len(_received) == MESSAGES: channel.stop_consuming() if start < time.time() - 2: assert False, "Test timed out" for x in xrange(0, MESSAGES): message = 'test_blocking_send:%i:%.4f' % (x, time.time()) _sent.append(message) channel.basic_publish(exchange=exchange_name, routing_key=routing_key, body=message, properties=pika.BasicProperties( content_type="text/plain", delivery_mode=1)) # Loop while we get messages (for 2 seconds) start = time.time() # This is blocking channel.basic_consume(consumer=_on_message, queue=queue_name, no_ack=True) connection.close() # Check our results if len(_sent) != MESSAGES: assert False, "We did not send the expected qty of messages: %i" %\ len(_sent) if len(_received) != MESSAGES: assert False, "Did not receive the expected qty of messages: %i" %\ len(_received) for message in _received: if message not in _sent: assert False, 'Received a message we did not send.' for message in _sent: if message not in _received: assert False, 'Sent a message we did not receive.'
class RabbitObj(object): ''' This is the class inherited by all other objects that work with RabbitMQ within CUFF. ''' def __init__(self, host='localhost', port=spec.PORT, virtual_host='/', credentials=None, channel_max=0, frame_max=spec.FRAME_MAX_SIZE, heartbeat=False): ''' Initializes all the connection parameters and stores them in a ConnectionParameters object at self.conn_params. Parameters: - host: Hostname or IP Address to connect to, defaults to localhost. - port: TCP port to connect to, defaults to 5672 - virtual_host: RabbitMQ virtual host to use, defaults to / - credentials: A instance of a credentials class to authenticate with. Defaults to PlainCredentials for the guest user. - channel_max: Maximum number of channels to allow, defaults to 0 for None - frame_max: The maximum byte size for an AMQP frame. Defaults to 131072 - heartbeat: Turn heartbeat checking on or off. Defaults to False. ''' # Send the values to a ConnectionParameters object, which has built-in # type-checking if isinstance(host, unicode): host = str(host) self.conn_params = pika.ConnectionParameters(host, port, virtual_host, credentials, channel_max, frame_max, heartbeat) self.connection = None self.channel = None self.init_callback = None self.exchange = None self.routing_key = None self.queue_created = False self.queue_name = None self.no_ack = False self._debug_prefix = "" # The binding key is essentially the same thing as a routing key, only it is # used by receivers instead of senders. We set it to None here so that we can # detect on the fly if we have executed a queue_bind() for the binding key # before we attempt to receive messages. This is also how we prevent senders # from performing a queue_bind(), since they don't generally do that. self.binding_key = None def init_connection(self, callback, queue_name='', exchange='', exchange_type='topic', routing_key="#", blocking=False, user_id=None): ''' Handles all connection, channel, etc. issues involved in fully connecting to the RabbitMQ server. The function 'callback' is stored and called once all steps have successfully completed. Using this method forces all communication to go through the queue specified by 'queue'. ''' self.exchange = str(exchange) self.init_callback = callback if user_id != None: self.user_id = str(user_id) if exchange_type not in EXCHANGE_TYPES: raise ValueError("Exchange type must be one of: %s" % \ str(EXCHANGE_TYPES)) else: self.ex_type = str(exchange_type) if isinstance(queue_name, unicode): queue_name = str(queue_name) elif not isinstance(queue_name, str): raise TypeError("Queue must be a str, got %s instead" % \ str(type(queue_name))) self.queue_name = queue_name if isinstance(routing_key, unicode): routing_key = str(routing_key) elif not isinstance(routing_key, str): raise TypeError("Routing key must be a str, got %s instead" % \ str(type(queue_name))) self.routing_key = routing_key if DEBUG: print self._debug_prefix + "About to start connection...", # From here, determine how to initialize the connection further by # what type of exchange was requested. if self.ex_type == "topic": self._init_topic_conn() #elif self.ex_type == "direct": # self._init_direct_conn() elif not blocking: self._init_topic_conn() else: raise NotImplementedError("Only 'topic' and 'direct' exchange " \ "types are currently supported.") def _init_topic_conn(self): self.connection = SelectConnection(self.conn_params, self.on_connected) try: # Loop so we can communicate with RabbitMQ self.connection.ioloop.start() except KeyboardInterrupt: # Gracefully close the connection self.connection.close() # Loop until we're fully closed, will stop on its own self.connection.ioloop.start() else: if DEBUG: print "Connection attempt rejected. Goodbye." def _init_direct_conn(self): blocking = True self.connection = BlockingConnection(self.conn_params) self.on_connected(self.connection, blocking) self.channel = self.connection.channel() self.on_channel_open(self.channel) def on_connected(self, connection, blocking=False): ''' ''' if DEBUG: print self._debug_prefix + "Connected\n Host: %s\n " \ "Exchange: %s\n" % (self.conn_params.host, self.exchange) + \ self._debug_prefix + "Creating channel...", # These should always be the same, but just in case... if self.connection is not connection: # Adopt the new connection object self.connection = connection if not blocking: self.connection.channel(on_open_callback=self.on_channel_open) def on_channel_open(self, new_channel): ''' ''' if DEBUG: print "Created\n" + self._debug_prefix + \ "Declaring %s exchange: '%s'" % (self.ex_type, self.exchange) self.channel = new_channel self.channel.exchange_declare(exchange=self.exchange, durable=True, type=self.ex_type, callback=self.init_callback) #TODO: Mike, I added some correlation_id stuff here. # Using this TODO as a marker so you can find it def send_message(self, body, routing_key=None, correlation_id=None): ''' Sends a message to the exchange on the server to which a connection has already been established. Both parameters 'body' and 'routing_key' must be of type str, or a TypeError will be raised. If no routing_key is specified, it defaults to the routing key value specified at declaration. ''' if type(body) != str: raise TypeError("Parameter 'body' must be of type 'str', got " \ "'%s' instead." % type(body)) if routing_key == None: routing_key = self.routing_key if type(routing_key) != str: raise TypeError("Parameter 'routing_key' must be of type 'str', " \ "got '%s' instead." % type(routing_key)) #if INFO: print "\n" + self._debug_prefix + "Sending message on %s : %s : %s" % (self.exchange, self.queue_name, routing_key) # TODO: In the following, create a means of catching "unroutable" messages (necessary because the 'mandatory' # flag is set) props = pika.BasicProperties(delivery_mode=2, # Persistent messages user_id=str(self.user_id)) if DEBUG: print "[x] Sending message to %s" % (routing_key) self.channel.basic_publish(exchange=self.exchange, routing_key=routing_key, body=body, mandatory=True, properties=props, ) def create_queue(self, durable=True, exclusive=False, name=None, callback=None): ''' Typically don't change the default parameters. ''' if not name: name = self.queue_name if DEBUG: print self._debug_prefix + "Creating queue '%s'..." % name self.queue_created = True self.channel.queue_declare(durable=durable, exclusive=exclusive, queue=name, callback=callback) def bind_routing_key(self, frame): if DEBUG: print self._debug_prefix + "Binding queue '%s' to key '%s'" % \ (self.queue_name, self.routing_key) self.binding_key = self.routing_key # Signals that we've performed the queue binding self.channel.queue_bind(exchange=self.exchange, queue=self.queue_name, routing_key=self.binding_key, callback=self._rcv()) def receive_message(self, callback=None, no_ack=False, frame=None): ''' Retrieves a message from the channel and queue specified in earlier initialization code. The callback function defaults to self.process_msg. Only under rare circumstances should no_ack be set to True, since this will not inform RabbitMQ that completed tasks can be removed from the queue. ''' if DEBUG: print self._debug_prefix + "Preparing to receive" # Set the callback function for processing received messages if callback == None: self.rcv_callback = self.process_msg else: self.rcv_callback = callback self.no_ack = no_ack # Check the queue, declare if necessary if not self.queue_created: self.create_queue(callback=self.bind_routing_key) # Check the binding key, bind the channel if necessary elif self.binding_key == None: if frame == None: raise TypeError("Parameter 'frame' must not be None if the " \ "routing key has not yet been bound.") self.bind_routing_key(frame) # self._rcv() is specified as the callback in this method else: self._rcv() def _rcv(self): self.channel.basic_qos(prefetch_count=1) if DEBUG: print self._debug_prefix + "Consuming..." self.channel.basic_consume(consumer_callback=self.rcv_callback, queue=self.queue_name, no_ack=self.no_ack) def ack(self, tag): self.channel.basic_ack(delivery_tag=tag) def reject(self, tag): self.channel.basic_reject(delivery_tag=tag) def process_msg(self, channel, method, header, body): raise NotImplementedError
env=os.environ) try: sts = pop.wait() #wait for child to terminate and get status except Exception, e: print str(e) status = pop.returncode #print "returncode is:",status stdOut = pop.stdout.read() stdErr = pop.stderr.read() for line in stdOut.split('\n'): if line.startswith("error_queue"): COUNT = int(line.split()[1]) break print "Total number of messages in error_queue:", COUNT if COUNT == 0: sys.exit() # Connect to RabbitMQ host = (len(sys.argv) > 1) and sys.argv[1] or '127.0.0.1' CONNECTION = BlockingConnection(ConnectionParameters(host)) # Open the channel channel = CONNECTION.channel() # loop for method_frame, properties, body in channel.consume('error_queue'): if handle_delivery(channel, method_frame, body): break channel.stop_consuming() channel.close() CONNECTION.close()
class AMQPLink(object): exceptions = pika.exceptions class PikaError(Exception): pass link = None tx = True def __init__( self, host, auth, exchange, heartbeat=False, reconnect_delays=5, libc_gethostbyname=None, log=None ): self.log = log or logging.getLogger('amqp') if heartbeat: raise NotImplementedError self.host, self.auth, self.exchange, self.heartbeat,\ self.libc_gethostbyname = host, auth, exchange, heartbeat, libc_gethostbyname if isinstance(reconnect_delays, (int, float)): reconnect_delays = [reconnect_delays] self.reconnect_delays, self.reconnect_info = reconnect_delays, None self.connect() # mainly to notify if it fails at start def schema_init(self): exchange = self.exchange.copy() self.ch.exchange_declare(exchange=exchange.pop('name'), **exchange) def _error_callback(self, msg, *argz, **kwz): raise kwz.pop('err', self.PikaError)(msg) def connect(self): host = self.host if self.libc_gethostbyname: host = self.libc_gethostbyname(self.host) while True: if self.link and self.link.is_open: try: self.link.close() except: pass try: self.log.debug('Connecting to AMQP broker ({})'.format(host)) self.link = BlockingConnection(ConnectionParameters( host, heartbeat=self.heartbeat, credentials=PlainCredentials(*self.auth) )) # Even with BlockingConnection adapter, # pika doesn't raise errors, unless you set callbacks to do that self.link.set_backpressure_multiplier(2) self.link.add_backpressure_callback( ft.partial(self._error_callback, 'timeout') ) self.link.add_on_close_callback( ft.partial(self._error_callback, 'closed/error') ) self.ch = self.link.channel() self.schema_init() if self.tx: self.ch.tx_select() # forces flush except (self.PikaError, socket.error) as err: self.log.exception('Connection to AMQP broker has failed: {}'.format(err)) delay = self.reconnect_info and self.reconnect_info[0] # first delay is 0 if delay: self.log.debug('Will retry connection in {}s'.format(delay)) sleep(delay) self.reconnect_info = self.reconnect_info or self.reconnect_delays if len(self.reconnect_info) > 1: self.reconnect_info = self.reconnect_info[1:] else: self.reconnect_info = None break
def test_blocking_consume_clean_close(): # Connect to RabbitMQ connection = BlockingConnection(support.PARAMETERS) # Open the channel channel = connection.channel() # Declare the exchange exchange_name = support.tools.test_queue_name('blocking_exchange') frame = channel.exchange_declare(exchange=exchange_name, type="direct", auto_delete=True) if not isinstance(frame.method, Exchange.DeclareOk): assert False, \ "Did not receive Exchange.DeclareOk from channel.exchange_declare" # Declare the queue queue_name = support.tools.test_queue_name('blocking_consume') frame = channel.queue_declare(queue=queue_name, durable=False, exclusive=True, auto_delete=True) if not isinstance(frame.method, Queue.DeclareOk): assert False, \ "Did not receive Queue.DeclareOk from channel.queue_declare" routing_key = "%s.%s" % (exchange_name, queue_name) frame = channel.queue_bind(queue=queue_name, exchange=exchange_name, routing_key=routing_key) if not isinstance(frame.method, Queue.BindOk): assert False, \ "Did not receive Queue.BindOk from channel.queue_bind" _sent = [] _received = [] def _on_message(channel, method, header, body): _received.append(body) if len(_received) == MESSAGES: return connection.close() if start < time() - MAX_DURATION: assert False, "Test timed out" print "Here" for x in xrange(0, MESSAGES): message = 'test_blocking_send:%i:%.4f' % (x, time()) _sent.append(message) channel.basic_publish(exchange=exchange_name, routing_key=routing_key, body=message, properties=BasicProperties( content_type="text/plain", delivery_mode=1)) # Loop while we get messages (for 2 seconds) start = time() channel.basic_consume(consumer_callback=_on_message, queue=queue_name, no_ack=True) # This is blocking channel.start_consuming() # Check our results if len(_sent) != MESSAGES: assert False, "We did not send the expected qty of messages: %i" %\ len(_sent) if len(_received) != MESSAGES: assert False, "Did not receive the expected qty of messages: %i" %\ len(_received) for message in _received: if message not in _sent: assert False, 'Received a message we did not send.' for message in _sent: if message not in _received: assert False, 'Sent a message we did not receive.'
if RUN_NUMBER is None: print "Run number (-r) must be specified" if FANOUT is None: print "Fanout (-f) must be specified" if DEPTH is None: print "Depth (-d) must be specified" if NUM_SITES is None: print "Num sites (-n) must be specified" if None in [RUN_NUMBER, FANOUT, DEPTH, NUM_SITES]: exit() # Connect to RabbitMQ TARGET_RMQ_SERVER = "ldr.myvnc.com" if DEBUG else "noddy.cs.berkeley.edu" parameters = pika.ConnectionParameters(TARGET_RMQ_SERVER) rmq_connection = BlockingConnection(parameters) rmq_channel = rmq_connection.channel() rmq_channel.queue_declare(queue="pages", durable=True, exclusive=False, auto_delete=False) reader = csv.reader(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'top-1m.csv'), 'rb')) n = 1 for row in reader: if n < OFFSET: n += 1 continue url = "http://%s" % row[1] command_data = { 'run': RUN_NUMBER, 'url': url,
""" import sys import time from pika.adapters import BlockingConnection from pika.connection import ConnectionParameters if __name__ == '__main__': # Connect to RabbitMQ host = (len(sys.argv) > 1) and sys.argv[1] or '127.0.0.1' connection = BlockingConnection(ConnectionParameters(host)) # Open the channel channel = connection.channel() # Declare the queue channel.queue_declare(queue="test", durable=True, exclusive=False, auto_delete=False) # Initialize our timers and loop until external influence stops us while connection.is_open: # Call basic get which returns the 3 frame types method, header, body = channel.basic_get(queue="test") # It can be empty if the queue is empty so don't do anything if method.NAME == 'Basic.GetEmpty':
'="foo">Bar</other><value>%i</value></node></document>' ) YAML_VALUE = """%%YAML 1.2 --- Application: poll_interval: 10.0 log_stats: True name: Example value: %i """ if __name__ == "__main__": connection = BlockingConnection(ConnectionParameters()) # Open the channel channel = connection.channel() channel.exchange_declare(exchange="example") # Declare the queue channel.queue_declare(queue="generated_messages", durable=True, exclusive=False, auto_delete=False) channel.queue_bind(exchange="example", queue="generated_messages", routing_key="rejected_example") channel.queue_declare(queue="consumer_replies", durable=True, exclusive=False, auto_delete=False) channel.queue_bind(exchange="example", queue="consumer_replies", routing_key="rejected_reply") # Initialize our timers and loop until external influence stops us for iteration in xrange(0, MESSAGE_COUNT): msg_type = random.randint(1, 4)
class Queue(object): """ A class for performing both sends and receives on a RabbitMQ server, using the pika library. """ def __init__(self): self.declared_channels = [] self.logger = getLogger(__name__) def set_logger(self, logger): self.logger = logger def declare_exchange(self, exchange_name, queue_options, exchange_type="topic"): try: self.channel.exchange_declare(exchange=exchange_name, type=exchange_type, **queue_options) except AMQPChannelError: pass def bind_queue(self, exchange, queue_name, routing_key=None): kwargs = dict( exchange=exchange, queue=queue_name ) if routing_key is not None: kwargs['routing_key'] = routing_key bind = self.channel.queue_bind(**kwargs) def connect(self, connection_params): """ Performs the actual connection to the RabbitMQ server. :param connection_params: The connection parameters. See pika.ConnectionParameters for available properties :type connection_params: dict """ connect_params_obj = ConnectionParameters(**connection_params) self.connection = BlockingConnection(connect_params_obj) self.channel = self.connection.channel() def declare_queue(self, queue_name=None, queue_options={}): """ Declares a queue on the server. A list of declared is kept so we do not redeclare an existing queue. :param queue: Name of the queue to declare :type queue: string :param queue_options: Options for the queue, for details see pika.spec.DriverMixin.queue_declare :type queue_options: dict """ queue_params = queue_options.copy() if queue_name not in self.declared_channels: if queue_name is not None: queue_params['queue'] = queue_name queue = self.channel.queue_declare(**queue_params) if queue_name is None: queue_name = queue.method.queue self.declared_channels.append(queue_name) return queue_name def disconnect(self): """ Drop the connection to the RabbitMQ server """ self.connection.close() def send(self, exchange_name, routing_key, body, exchange_type="topic"): """ Put a message into the specified queue :param exchange_name: The name of the exchange to use :type exchange_name: str :param routing_key: The routing key to be used for this message :type routing_key: str :param body: The actual message body :type body: str :param exchange_type: Which type of exchange to use :type body: str """ if not isinstance(body, str): body = json.dumps(body) try: self.channel except NameError: self.logger.error("You must connect first!") self.channel.basic_publish(exchange=exchange_name, routing_key=routing_key, body=body, properties=BasicProperties(content_type="text/plain", delivery_mode=2) ) self.logger.info( "Message sent!") def receive(self, queue_name, callback, timeout=None, limit=None, finish_on_empty=True, finished_callback=None, exchange_name='', exchange_type="topic", routing_key="#", queue_options={}): """ Receive messages from a given queue, and pass them to a callback function. This method will keep returning messages until one of the named break conditions is reached: * timeout: Stop receiving messages after x seconds * limit: Stop receiving messages after x messages have been consumed * finish_on_empty: If true, stop receiving messages when the queue is empty An optional finished callback can be called when the break condition has been reached. :param queue_name: The name of the queue to receive messages from :type queue_name: str :param callback: The callback to pass each message to :type callback: Callable :param timeout: The number of seconds before we stop receiving :type timeout: int :param limit: The number of messages received before we stop receiving :type limit: int :param finish_on_empty: Whether to finish when the queue is empty :type finish_on_empty: bool :param finished_callback: What to call when when we have finished receiving messages :type finished_callback: Callable :param queue_options: Options for the queue, for details see pika.spec.DriverMixin.queue_declare :type queue_options: dict """ self.callback = callback self.timeout = timeout self.count = 0 self.limit = limit self.finish_on_empty = finish_on_empty if timeout is not None: self.end_time = datetime.datetime.now() + datetime.timedelta(seconds=timeout) else: self.end_time = None keep_reading = True try: self.channel except NameError: self.logger.error( "You must connect first!") queue_name = self.declare_queue(queue_name, queue_options) if exchange_name: self.declare_exchange(exchange_type=exchange_type, exchange_name=exchange_name, queue_options=queue_options) self.bind_queue(exchange_name, queue_name, routing_key=routing_key) TIME_FORMAT = "%y-%m-%d %H:%M:%S" while keep_reading: if self.end_time is not None: now = datetime.datetime.now() if now > self.end_time: keep_reading = False self.logger.debug( "Past timeout, leaving!") else: self.logger.debug( "{0} is before {1}, as you were".format( now.strftime(TIME_FORMAT), self.end_time.strftime(TIME_FORMAT)) ) if self.limit is not None: if self.count >= self.limit: keep_reading = False self.logger.debug( "Reached message limit of {0}, leaving!".format(self.limit)) else: self.logger.debug( "{0} is less than {1}, as you were".format(self.count, self.limit)) method, header_frame, body = self.channel.basic_get(queue=queue_name) # Pika changed the API for this. For Basic.GetEmpty # they now return None, None, None if method is None and header_frame is None and body is None: if self.finish_on_empty is True: keep_reading = False self.logger.debug("Queue empty, leaving!") else: sleep(1) continue else: self.count += 1 self.logger.debug( "Queue not empty, as you were") kwargs = dict( routing_key=method.routing_key, exchange=method.exchange ) try: callback(json.loads(body), **kwargs) except Exception, e: self.channel.basic_reject(delivery_tag=method.delivery_tag, requeue=True) self.logger.critical(e) raise e self.logger.debug("Acknowledging message {0}".format( method.delivery_tag)) self.channel.basic_ack(delivery_tag=method.delivery_tag) if finished_callback is not None: finished_callback()