def test_close_disconnects(self): c = Connection(transport=Transport).channel() conn1 = c.client.connection conn2 = c.subclient.connection c.close() self.assertTrue(conn1.disconnected) self.assertTrue(conn2.disconnected)
def connection_thread(url, results, hide_password=False): from oslo_config import cfg from oslo_messaging.transport import TransportURL from pika import exceptions as pika_exceptions from pika import URLParameters as PikaUrlParameters from pika import BlockingConnection as PikaBlockingConnection try: parsed_url = TransportURL.parse(cfg.CONF, url) if hide_password: url = re.sub(':+[^:@]+@', ':******@', url) except Exception as e: results.append({'url': url, 'exception': e}) else: test_url, driver = parse_test_url(parsed_url) try: if driver == 'kombu': connection = Connection(test_url) connection.connect() connection.close() elif driver == 'pika': params = PikaUrlParameters(test_url) params.socket_timeout = 5 conn = PikaBlockingConnection(params) conn.close() except (OSError, pika_exceptions.ConnectionClosed): results.append({'url': url, 'exception': _('Url not reachable')}) except (AccessRefused, pika_exceptions.ProbableAuthenticationError): results.append({ 'url': url, 'exception': _('Credentials incorrect') }) except Exception as e: results.append({'url': url, 'exception': force_text(e)}) else: results.append({'url': url})
def test_db_port(self): c1 = Connection(port=None, transport=Transport).channel() self.assertEqual(c1.client.port, Transport.default_port) c1.close() c2 = Connection(port=9999, transport=Transport).channel() self.assertEqual(c2.client.port, 9999) c2.close()
def test_connection(host, port, user_id, password, virt_host, exchange_name, queue_name): """ Test a connection to an exchange on a virtual host """ connection = None connected = False success = False try: # Connect to the virtual host - will raise exception if it fails. connection = Connection(host, user_id, password, virt_host, port) connection.connect() connected = connection.connected if connected: # Check whether exchange exists - will raise exception if it fails. exchange = Exchange(exchange_name, channel=connection, type='topic', durable=False, passive=True) exchange.declare() # Check whether the queue exists - will raise exception if it # fails. rpc_receive_queue = Queue(queue_name, durable=True, exchange=exchange, channel=connection) rpc_receive_queue.queue_declare(passive=True) success = True except Exception as e: DLOG.info("Unable to connect to virt_host %s, exchange %s, error: %s" % (virt_host, exchange_name, e)) finally: if connected: connection.close() return success
class MessageQueuePipeline(object): """Emit processed items to a RabbitMQ exchange/queue""" def __init__(self, host_name, port, userid, password, virtual_host, encoder_class): self.connection = Connection(hostname=host_name, port=port, userid=userid, password=password, virtual_host=virtual_host) self.encoder = encoder_class() dispatcher.connect(self.spider_opened, signals.spider_opened) dispatcher.connect(self.spider_closed, signals.spider_closed) @classmethod def from_settings(cls, settings): host_name = settings.get('BROKER_HOST') port = settings.get('BROKER_PORT') userid = settings.get('BROKER_USERID') password = settings.get('BROKER_PASSWORD') virtual_host = settings.get('BROKER_VIRTUAL_HOST') encoder_class = settings.get('MESSAGE_Q_SERIALIZER', ScrapyJSONEncoder) return cls(host_name, port, userid, password, virtual_host, encoder_class) def spider_opened(self, spider): self.queue = self.connection.SimpleQueue(spider.name) def spider_closed(self, spider): self.queue.close() self.connection.close() def process_item(self, item, spider): return deferToThread(self._process_item, item, spider) def _process_item(self, item, spider): self.queue.put(self.encoder.encode(dict(item))) return item
def test_close_ResponseError(self): c = Connection(transport=Transport).channel() c.client.bgsave_raises_ResponseError = True c.close()
def test_close_poller_not_active(self): c = Connection(transport=Transport).channel() cycle = c.connection.cycle c.client.connection c.close() self.assertNotIn(c, cycle._channels)
class test_Redis(TestCase): def setUp(self): self.connection = Connection(transport=Transport) self.exchange = Exchange('test_Redis', type='direct') self.queue = Queue('test_Redis', self.exchange, 'test_Redis') def tearDown(self): self.connection.close() def test_publish__get(self): channel = self.connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') self.queue(channel).declare() producer.publish({'hello': 'world'}) self.assertDictEqual(self.queue(channel).get().payload, {'hello': 'world'}) self.assertIsNone(self.queue(channel).get()) self.assertIsNone(self.queue(channel).get()) self.assertIsNone(self.queue(channel).get()) def test_publish__consume(self): connection = Connection(transport=Transport) channel = connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') consumer = Consumer(channel, self.queue) producer.publish({'hello2': 'world2'}) _received = [] def callback(message_data, message): _received.append(message_data) message.ack() consumer.register_callback(callback) consumer.consume() self.assertIn(channel, channel.connection.cycle._channels) try: connection.drain_events(timeout=1) self.assertTrue(_received) with self.assertRaises(socket.timeout): connection.drain_events(timeout=0.01) finally: channel.close() def test_purge(self): channel = self.connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') self.queue(channel).declare() for i in range(10): producer.publish({'hello': 'world-%s' % (i, )}) self.assertEqual(channel._size('test_Redis'), 10) self.assertEqual(self.queue(channel).purge(), 10) channel.close() def test_db_values(self): c1 = Connection(virtual_host=1, transport=Transport).channel() self.assertEqual(c1.client.db, 1) c2 = Connection(virtual_host='1', transport=Transport).channel() self.assertEqual(c2.client.db, 1) c3 = Connection(virtual_host='/1', transport=Transport).channel() self.assertEqual(c3.client.db, 1) with self.assertRaises(Exception): Connection(virtual_host='/foo', transport=Transport).channel() def test_db_port(self): c1 = Connection(port=None, transport=Transport).channel() self.assertEqual(c1.client.port, Transport.default_port) c1.close() c2 = Connection(port=9999, transport=Transport).channel() self.assertEqual(c2.client.port, 9999) c2.close() def test_close_poller_not_active(self): c = Connection(transport=Transport).channel() cycle = c.connection.cycle c.client.connection c.close() self.assertNotIn(c, cycle._channels) def test_close_ResponseError(self): c = Connection(transport=Transport).channel() c.client.bgsave_raises_ResponseError = True c.close() def test_close_disconnects(self): c = Connection(transport=Transport).channel() conn1 = c.client.connection conn2 = c.subclient.connection c.close() self.assertTrue(conn1.disconnected) self.assertTrue(conn2.disconnected) def test_get__Empty(self): channel = self.connection.channel() with self.assertRaises(Empty): channel._get('does-not-exist') channel.close() def test_get_client(self): myredis, exceptions = _redis_modules() @module_exists(myredis, exceptions) def _do_test(): conn = Connection(transport=Transport) chan = conn.channel() self.assertTrue(chan.Client) self.assertTrue(chan.ResponseError) self.assertTrue(conn.transport.connection_errors) self.assertTrue(conn.transport.channel_errors) _do_test()
class RabbitMq: "manage rabbitmq." def __init__(self,mq_server): "" self.connection = Connection(mq_server) self.chan = self.connection.channel() # 关闭 def mq_close(self): "" self.connection.close() self.chan.close() # 声明queue def queue_declare(self,**kwargs): "" queue_name = kwargs.pop('queue_name', None) durable_flag = kwargs.pop('durable_flag', True) exclusive_flag = kwargs.pop('exclusive_flag', False) auto_delete_flag = kwargs.pop('auto_delete_flag', False) self.chan.queue_declare(queue=queue_name, durable=durable_flag, exclusive=exclusive_flag, auto_delete=auto_delete_flag) # 声明exchange def exchange_declare(self,**kwargs): "" exchange_name = kwargs.pop('exchange_name', None) mq_type = kwargs.pop('mq_type', None) durable_flag = kwargs.pop('durable_flag', True) auto_delete_flag = kwargs.pop('auto_delete_flag', False) self.chan.exchange_declare(exchange=exchange_name, type=mq_type, durable=durable_flag, auto_delete=auto_delete_flag) # 绑定queue 和 exchange def queue_bind(self,**kwargs): "" queue_name = kwargs.pop('queue_name', None) exchange_name = kwargs.pop('exchange_name', None) routing_key = kwargs.pop('routing_key', None) self.chan.queue_bind(queue=queue_name, exchange=exchange_name, routing_key=routing_key) # 发送消息 def mq_send(self,**kwargs): "" msg = kwargs.pop('msg', None) exchange_name = kwargs.pop('exchange_name', None) routing_key = kwargs.pop('routing_key', None) message = amqp.Message(str(msg)) self.chan.basic_publish(message,exchange=exchange_name,routing_key=routing_key) #接收消息 def mq_receive(self,**kwargs): "" queue_name = kwargs.pop('queue_name', None) callback = kwargs.pop('callback', None) self.chan.basic_consume(callback=callback,queue=queue_name,no_ack=True) while True: self.chan.wait()