示例#1
0
def consume_messages(amqp_string):
    conn = BrokerConnection(amqp_string)
    queue = conn.SimpleQueue("eventstream")

    while True:
        try:
            message = queue.get(block=False, timeout=1)
        except Empty:
            break
        else:
            message.ack()
            pl = message.payload
            log_message = {
                'state': pl['state'],
                'state_msg': pl['state_msg'],
                'host': pl['host'],
                'body': pl['body'],
                'timestamp': pl['timestamp'],
                'html': loader.load("message.html").generate(message=pl)
            }
            MessageHandler.update_cache(log_message)
            MessageHandler.send_updates(log_message)

    queue.close()
    conn.close()
def wait_many(timeout=1):

    #: Create connection
    #: If hostname, userid, password and virtual_host is not specified
    #: the values below are the default, but listed here so it can
    #: be easily changed.
    connection = BrokerConnection("amqp://*****:*****@localhost:5672//")

    #: SimpleQueue mimics the interface of the Python Queue module.
    #: First argument can either be a queue name or a kombu.Queue object.
    #: If a name, then the queue will be declared with the name as the queue
    #: name, exchange name and routing key.
    queue = connection.SimpleQueue("kombu_demo")

    while True:
        try:
            message = queue.get(block=False, timeout=timeout)
        except Empty:
            break
        else:
            spawn(message.ack)
            print(message.payload)

    queue.close()
    connection.close()
示例#3
0
文件: views.py 项目: sp00/kral
def exchange_send(data,exchange):
    try:
        connection = BrokerConnection()
        channel = connection.channel()
        producer = Producer(channel, Exchange(exchange, type="fanout"))
        producer.publish(data)
        channel.close()
        connection.close()
    except Exception, error:
        print(error)
示例#4
0
 def _establish_connection(self, conn: BrokerConnection) -> None:
     """
     We don't use a pool here. We only have one consumer connection per process, so
     we get no value from a pool, and we want to use a heartbeat to keep the consumer
     collection alive, which does not work with a pool
     :return: the connection to the transport
     """
     try:
         self._logger.debug("Establishing connection.")
         self._conn = conn.ensure_connection(max_retries=3)
         self._logger.debug('Got connection: %s', conn.as_uri())
     except kombu_exceptions.OperationalError as oe:
         self._logger.error("Error connecting to RMQ, could not retry %s", oe)
         # Try to clean up the mess
         if self._conn is not None:
             self._conn.close()
         else:
             conn.close()
def main():
    conn = BrokerConnection("amqp://*****:*****@localhost:5672//")
    queue = conn.SimpleQueue("eventstream")
    queue_options = {'serializer': 'json', 'compression': 'zlib'}

    host = socket.gethostname()

    while True:
        queue.put({
            'state': random.choice(bootstrap_labels),
            'state_msg': 'Hello',
            'host': host,
            'timestamp': datetime.datetime.now().isoformat(),
            'body': 'foobar'}, **queue_options)
        time.sleep(random.random() / 8.0)

    queue.close()
    conn.close()
def send_many(n):

    #: Create connection
    #: If hostname, userid, password and virtual_host is not specified
    #: the values below are the default, but listed here so it can
    #: be easily changed.
    connection = BrokerConnection("amqp://*****:*****@localhost:5672//")

    #: SimpleQueue mimics the interface of the Python Queue module.
    #: First argument can either be a queue name or a kombu.Queue object.
    #: If a name, then the queue will be declared with the name as the queue
    #: name, exchange name and routing key.
    queue = connection.SimpleQueue("kombu_demo")

    def send_message(i):
        queue.put({"hello": "world%s" % (i, )})

    pool = eventlet.GreenPool(10)
    for i in xrange(n):
        pool.spawn(send_message, i)
    pool.waitall()

    queue.close()
    connection.close()
示例#7
0
def send_many(n):

    #: Create connection
    #: If hostname, userid, password and virtual_host is not specified
    #: the values below are the default, but listed here so it can
    #: be easily changed.
    connection = BrokerConnection("amqp://*****:*****@localhost:5672//")

    #: SimpleQueue mimics the interface of the Python Queue module.
    #: First argument can either be a queue name or a kombu.Queue object.
    #: If a name, then the queue will be declared with the name as the queue
    #: name, exchange name and routing key.
    queue = connection.SimpleQueue("kombu_demo")

    def send_message(i):
        queue.put({"hello": "world%s" % (i, )})

    pool = eventlet.GreenPool(10)
    for i in xrange(n):
        pool.spawn(send_message, i)
    pool.waitall()

    queue.close()
    connection.close()
示例#8
0
class test_pika(unittest.TestCase):

    def purge(self, names):
        chan = self.connection.channel()
        map(chan.queue_purge, names)

    def setUp(self):
        self.connection = BrokerConnection(transport="pika")
        try:
            self.connection.connect()
        except socket.error:
            self.connected = False
        else:
            self.connected = True
        self.exchange = Exchange("tamqplib", "direct")
        self.queue = Queue("tamqplib", self.exchange, "tamqplib")

    def test_produce__consume(self):
        if not self.connected:
            raise SkipTest("Broker not running.")
        chan1 = self.connection.channel()
        producer = Producer(chan1, self.exchange)

        producer.publish({"foo": "bar"}, routing_key="tamqplib")
        chan1.close()

        chan2 = self.connection.channel()
        consumer = Consumer(chan2, self.queue)
        message = consumeN(self.connection, consumer)
        self.assertDictEqual(message[0], {"foo": "bar"})
        chan2.close()
        self.purge(["tamqplib"])

    def test_produce__consume_multiple(self):
        if not self.connected:
            raise SkipTest("Broker not running.")
        chan1 = self.connection.channel()
        producer = Producer(chan1, self.exchange)
        b1 = Queue("pyamqplib.b1", self.exchange, "b1")
        b2 = Queue("pyamqplib.b2", self.exchange, "b2")
        b3 = Queue("pyamqplib.b3", self.exchange, "b3")

        producer.publish("b1", routing_key="b1")
        producer.publish("b2", routing_key="b2")
        producer.publish("b3", routing_key="b3")
        chan1.close()

        chan2 = self.connection.channel()
        consumer = Consumer(chan2, [b1, b2, b3])
        messages = consumeN(self.connection, consumer, 3)
        self.assertItemsEqual(messages, ["b1", "b2", "b3"])
        chan2.close()
        self.purge(["pyamqplib.b1", "pyamqplib.b2", "pyamqplib.b3"])

    def test_timeout(self):
        if not self.connected:
            raise SkipTest("Broker not running.")
        chan = self.connection.channel()
        self.purge([self.queue.name])
        consumer = Consumer(chan, self.queue)
        self.assertRaises(socket.timeout, self.connection.drain_events,
                timeout=0.3)
        consumer.cancel()

    def test_basic_get(self):
        chan1 = self.connection.channel()
        producer = Producer(chan1, self.exchange)
        producer.publish({"basic.get": "this"}, routing_key="basic_get")
        chan1.close()

        chan2 = self.connection.channel()
        queue = Queue("amqplib_basic_get", self.exchange, "basic_get")
        queue = queue(chan2)
        queue.declare()
        for i in range(50):
            m = queue.get()
            if m:
                break
            time.sleep(0.1)
        self.assertEqual(m.payload, {"basic.get": "this"})
        chan2.close()

    def tearDown(self):
        if self.connected:
            self.connection.close()
示例#9
0
class Connection:
    connection = None

    def __init__(self, url):
        self.url = url
        self.__connection = None
        self.__running = True
        self.channel = None
        self.sleep_time = 10
        self.reconnect(url)

    @staticmethod
    def get_instance():
        return Connection.connection

    def __connect(self):
        self.__connection = BrokerConnection(self.url)
        self.channel = self.get_channel()

        self.rpc_factory = rpc.RpcFactory(self.channel)
        self.publisher_factory = publisher.PublisherFactory(self.channel)
        self.consumer_factory = consumer.ConsumerFactory(self.channel)

        self.__running = True
        Connection.connection = self

    def get_broker_connection(self):
        if self.__connection is None:
            self.reconnect(self.url)

        return self.__connection

    def get_channel(self):
        if self.channel is None:
            self.channel = self.get_new_channel()
        return self.channel

    def get_new_channel(self):
        if self.__connection is None:
            self.reconnect(self.url)
        return self.__connection.channel()

    def get_rpc_factory(self):
        return self.rpc_factory

    def reconnect(self, url=None):
        cc.acquire()
        if self.__connection is not None:
            self.release()

        if url is not None:
            self.url = url

        logger.debug("reconnect connection")
        attempt = 0
        while True:
            try:
                self.__connect()
                cc.release()
                return
            except Exception as e:
                logging.exception(e)

            logging.debug("retry again in %s s" % self.sleep_time)
            time.sleep(self.sleep_time)
        cc.release()

    def drain_events(self):
        self.__connection.drain_events()

    def release(self):
        Connection.connection = None
        self.__running = False
        self.__connection.release()
        self.__connection.close()
        self.channel = None
        self.__connection = None
示例#10
0
connection = BrokerConnection(
                hostname='rh2.dev.novagile.fr',
                transport="redis",
                virtual_host=0,
                port=6379)

print "Connection Producer to Redis"
connection.connect()

queue = connection.SimpleQueue("pdf_to_jpg")

for f in files:
    # open as binary
    my_file = open(f, "rb")
    my_file.seek(0)
    my_file_bcontent = my_file.read()
    my_file.close()

    # Push !
    queue.put({"file_content": my_file_bcontent,
                "file_name": f,
                "hostname":  gethostname(),
                "timestamp": time()},
                serializer="pickle",
                compression=None)

    my_file.close()
    print "Pushed on queue pdf_to_jpg"

connection.close()
示例#11
0
class SimpleBase(TestCase):
    abstract = True

    def Queue(self, name, *args, **kwargs):
        q = name
        if not isinstance(q, Queue):
            q = self.__class__.__name__
            if name:
                q = "%s.%s" % (q, name)
        return self._Queue(q, *args, **kwargs)

    def _Queue(self, *args, **kwargs):
        raise NotImplementedError()

    def setUp(self):
        if not self.abstract:
            self.connection = BrokerConnection(transport="memory")
            self.q = self.Queue(None, no_ack=True)

    def tearDown(self):
        if not self.abstract:
            self.q.close()
            self.connection.close()

    def test_produce__consume(self):
        if self.abstract:
            return
        q = self.Queue("test_produce__consume", no_ack=True)

        q.put({"hello": "Simple"})

        self.assertEqual(q.get(timeout=1).payload, {"hello": "Simple"})
        with self.assertRaises(Empty):
            q.get(timeout=0.1)

    def test_produce__basic_get(self):
        if self.abstract:
            return
        q = self.Queue("test_produce__basic_get", no_ack=True)
        q.put({"hello": "SimpleSync"})
        self.assertEqual(q.get_nowait().payload, {"hello": "SimpleSync"})
        with self.assertRaises(Empty):
            q.get_nowait()

        q.put({"hello": "SimpleSync"})
        self.assertEqual(q.get(block=False).payload, {"hello": "SimpleSync"})
        with self.assertRaises(Empty):
            q.get(block=False)

    def test_clear(self):
        if self.abstract:
            return
        q = self.Queue("test_clear", no_ack=True)

        for i in range(10):
            q.put({"hello": "SimplePurge%d" % (i, )})

        self.assertEqual(q.clear(), 10)

    def test_enter_exit(self):
        if self.abstract:
            return
        q = self.Queue("test_enter_exit")
        q.close = Mock()

        self.assertIs(q.__enter__(), q)
        q.__exit__()
        q.close.assert_called_with()

    def test_qsize(self):
        if self.abstract:
            return
        q = self.Queue("test_clear", no_ack=True)

        for i in range(10):
            q.put({"hello": "SimplePurge%d" % (i, )})

        self.assertEqual(q.qsize(), 10)
        self.assertEqual(len(q), 10)

    def test_autoclose(self):
        if self.abstract:
            return
        channel = self.connection.channel()
        q = self.Queue("test_autoclose", no_ack=True, channel=channel)
        q.close()

    def test_custom_Queue(self):
        if self.abstract:
            return
        n = self.__class__.__name__
        exchange = Exchange("%s-test.custom.Queue" % (n, ))
        queue = Queue("%s-test.custom.Queue" % (n, ),
                      exchange,
                      "my.routing.key")

        q = self.Queue(queue)
        self.assertEqual(q.consumer.queues[0], queue)
        q.close()

    def test_bool(self):
        if self.abstract:
            return
        q = self.Queue("test_nonzero")
        self.assertTrue(q)
示例#12
0
"""

Example that sends a single message and exits using the simple interface.

You can use `simple_receive.py` (or `complete_receive.py`) to receive the
message sent.

"""

from kombu import BrokerConnection

#: Create connection
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
connection = BrokerConnection(hostname="localhost",
                              userid="guest",
                              password="******",
                              virtual_host="/")

#: SimpleQueue mimics the interface of the Python Queue module.
#: First argument can either be a queue name or a kombu.Queue object.
#: If a name, then the queue will be declared with the name as the queue
#: name, exchange name and routing key.
queue = connection.SimpleQueue("kombu_demo")
queue.put({"hello": "world"}, serializer="json", compression="zlib")

# Always remember to close channels and connections.
queue.close()
connection.close()
示例#13
0
文件: server.py 项目: mkisto/callme
class Server(object):
	"""
	This Server class is used to provide an RPC server

	:keyword server_id: Id of the server
	:keyword amqp_host: The host of where the AMQP Broker is running.
	:keyword amqp_user: The username for the AMQP Broker.
	:keyword amqp_password: The password for the AMQP Broker.
	:keyword amqp_vhost: The virtual host of the AMQP Broker.
	:keyword amqp_port: The port of the AMQP Broker.
	:keyword ssl: Use SSL connection for the AMQP Broker.
	:keyword threaded: Use of multithreading. If set to true RPC call-execution
		will processed parallel (one thread per call) which dramatically improves
		performance.


	"""
	
	def __init__(self, 
				server_id,
				amqp_host='localhost', 
				amqp_user ='******',
				amqp_password='******',
				amqp_vhost='/',
				amqp_port=5672,
				ssl=False,
				threaded=False):
		self.logger = logging.getLogger('callme.server')
		self.logger.debug('Server ID: %s' % server_id)
		self.server_id = server_id
		self.threaded = threaded
		self.do_run = True
		self.is_stopped = True
		self.func_dict={}
		self.result_queue = queue.Queue()
		target_exchange = Exchange("server_"+server_id+"_ex", "direct", durable=False,
								auto_delete=True)	
		self.target_queue = Queue("server_"+server_id+"_queue", exchange=target_exchange, 
							auto_delete=True, durable=False)
		
		
		
		self.connection = BrokerConnection(hostname=amqp_host,
                              userid=amqp_user,
                              password=amqp_password,
                              virtual_host=amqp_vhost,
                              port=amqp_port,
                              ssl=ssl)
		try:
			self.connection.connect()
		except IOError:
			self.logger.critical("Connection Error: Probably AMQP User has not enough permissions")
			raise ConnectionError("Connection Error: Probably AMQP User has not enough permissions")
		
		self.channel = self.connection.channel()
		
		self.publish_connection = BrokerConnection(hostname=amqp_host,
                              userid=amqp_user,
                              password=amqp_password,
                              virtual_host=amqp_vhost,
                              port=amqp_port,
                              ssl=ssl)
		self.publish_channel = self.publish_connection.channel()
		
		# consume
		self.consumer = Consumer(self.channel, self.target_queue)
		if self.threaded == True:
			self.consumer.register_callback(self._on_request_threaded)
		else:
			self.consumer.register_callback(self._on_request)
		self.consumer.consume()
		
		self.logger.debug('Init done')
		
	def _on_request(self, body, message):
		"""
		This method is automatically called when a request is incoming. It 
		processes the incomming rpc calls in a serial manner (no multithreading)

		:param body: the body of the amqp message already unpickled by kombu
		:param message: the plain amqp kombu.message with aditional information
		"""
		self.logger.debug('Got Request')
		rpc_req = body
		
		if not isinstance(rpc_req, RpcRequest):
			self.logger.debug('Not an RpcRequest Instance')
			return
		
		self.logger.debug('Call func on Server %s' %self.server_id)
		try:
			self.logger.debug('corr_id: %s' % message.properties['correlation_id'])
			self.logger.debug('Call func with args %s' % repr(rpc_req.func_args))
			
			result = self.func_dict[rpc_req.func_name](*rpc_req.func_args)
			
			self.logger.debug('Result: %s' % repr(result))
			self.logger.debug('Build respnse')
			rpc_resp = RpcResponse(result)
		except Exception as e:
			self.logger.debug('exception happened')
			rpc_resp = RpcResponse(e, exception_raised=True)
			
		message.ack()
		
		self.logger.debug('Publish respnse')
		# producer 
		src_exchange = Exchange(message.properties['reply_to'], "direct", durable=False,
							auto_delete=True)
		self.producer = Producer(self.publish_channel, src_exchange, auto_declare=False)
		
		self.producer.publish(rpc_resp, serializer="pickle",
							correlation_id=message.properties['correlation_id'])
		
		self.logger.debug('acknowledge')
		


	def _on_request_threaded(self, body, message):
		"""
		This method is automatically called when a request is incoming and
		`threaded` set to `True`. It processes the incomming rpc calls in 
		a parallel manner (one thread for each request). A seperate Publisher
		thread is used to send back the results.

		:param body: the body of the amqp message already unpickled by kombu
		:param message: the plain amqp kombu.message with aditional information
		"""
		self.logger.debug('Got Request')
		rpc_req = body
		
		if not isinstance(rpc_req, RpcRequest):
			self.logger.debug('Not an RpcRequest Instance')
			return
		
		message.ack()
		self.logger.debug('acknowledge')
		
		def exec_func(body, message, result_queue):
			self.logger.debug('Call func on Server %s' %self.server_id)
			try:
				self.logger.debug('corr_id: %s' % message.properties['correlation_id'])
				self.logger.debug('Call func with args %s' % repr(rpc_req.func_args))
				
				result = self.func_dict[rpc_req.func_name](*rpc_req.func_args)
				
				self.logger.debug('Result: %s' % repr(result))
				self.logger.debug('Build respnse')
				rpc_resp = RpcResponse(result)
			except Exception as e:
				self.logger.debug('exception happened')
				rpc_resp = RpcResponse(e, exception_raised=True)
				
			result_queue.put(ResultSet(rpc_resp, 
									message.properties['correlation_id'],
									message.properties['reply_to']))
				
		p = Thread(target=exec_func, 
				name=message.properties['correlation_id'],
				args=(body, message, self.result_queue))
		p.start()
		
	
	def register_function(self, func, name):
		"""
		Registers a function as rpc function so that is accessible from the 
		proxy.
		
		:param func: The function we want to provide as rpc method
		:param name: The name with which the function is visible to the clients
		"""
		self.func_dict[name] = func
	
	def start(self):
		"""
		Starts the server. If `threaded` is `True` also starts the Publisher 
		thread.
		"""
		self.is_stopped = False
		if self.threaded == True:
			self.pub_thread = Publisher(self.result_queue, self.publish_channel)
			self.pub_thread.start()
			
		while self.do_run:
			try:
				self.logger.debug("drain_events: %s" % repr(self.do_run))
				self.connection.drain_events(timeout=1)
			except socket.timeout:
				self.logger.debug("do_run: %s" % repr(self.do_run))
			except:
				self.logger.debug("interrupt exception" )
				if self.threaded == True:
					self.pub_thread.stop()
				self.consumer.cancel()
				self.connection.close()
				self.publish_connection.close()
				self.is_stopped = True
				return
			
		if self.threaded == True:
			self.pub_thread.stop()
		self.logger.debug("Normal Exit" )
		self.consumer.cancel()
		self.connection.close()
		self.publish_connection.close()
		self.logger.debug("All closed" )
		self.is_stopped = True
		
	def stop(self):
		"""
		Stops the server.
		"""
		self.logger.debug('Stop server')
		self.do_run = False
		while not self.is_stopped:
			self.logger.debug('wait for stop')
			sleep(0.1)