def test_close_disconnects(self):
c = Connection(transport=Transport).channel()
conn1 = c.client.connection
conn2 = c.subclient.connection
c.close()
self.assertTrue(conn1.disconnected)
self.assertTrue(conn2.disconnected)
def test_close_disconnects(self):
c = Connection(transport=Transport).channel()
conn1 = c.client.connection
conn2 = c.subclient.connection
c.close()
self.assertTrue(conn1.disconnected)
self.assertTrue(conn2.disconnected)
def connection_thread(url, results, hide_password=False):
from oslo_config import cfg
from oslo_messaging.transport import TransportURL
from pika import exceptions as pika_exceptions
from pika import URLParameters as PikaUrlParameters
from pika import BlockingConnection as PikaBlockingConnection
try:
parsed_url = TransportURL.parse(cfg.CONF, url)
if hide_password:
url = re.sub(':+[^:@]+@', ':******@', url)
except Exception as e:
results.append({'url': url, 'exception': e})
else:
test_url, driver = parse_test_url(parsed_url)
try:
if driver == 'kombu':
connection = Connection(test_url)
connection.connect()
connection.close()
elif driver == 'pika':
params = PikaUrlParameters(test_url)
params.socket_timeout = 5
conn = PikaBlockingConnection(params)
conn.close()
except (OSError, pika_exceptions.ConnectionClosed):
results.append({'url': url, 'exception': _('Url not reachable')})
except (AccessRefused, pika_exceptions.ProbableAuthenticationError):
results.append({
'url': url,
'exception': _('Credentials incorrect')
})
except Exception as e:
results.append({'url': url, 'exception': force_text(e)})
else:
results.append({'url': url})
def test_db_port(self):
c1 = Connection(port=None, transport=Transport).channel()
self.assertEqual(c1.client.port, Transport.default_port)
c1.close()
c2 = Connection(port=9999, transport=Transport).channel()
self.assertEqual(c2.client.port, 9999)
c2.close()
def test_db_port(self):
c1 = Connection(port=None, transport=Transport).channel()
self.assertEqual(c1.client.port, Transport.default_port)
c1.close()
c2 = Connection(port=9999, transport=Transport).channel()
self.assertEqual(c2.client.port, 9999)
c2.close()
def test_connection(host, port, user_id, password, virt_host, exchange_name,
queue_name):
"""
Test a connection to an exchange on a virtual host
"""
connection = None
connected = False
success = False
try:
# Connect to the virtual host - will raise exception if it fails.
connection = Connection(host, user_id, password, virt_host, port)
connection.connect()
connected = connection.connected
if connected:
# Check whether exchange exists - will raise exception if it fails.
exchange = Exchange(exchange_name,
channel=connection,
type='topic',
durable=False,
passive=True)
exchange.declare()
# Check whether the queue exists - will raise exception if it
# fails.
rpc_receive_queue = Queue(queue_name,
durable=True,
exchange=exchange,
channel=connection)
rpc_receive_queue.queue_declare(passive=True)
success = True
except Exception as e:
DLOG.info("Unable to connect to virt_host %s, exchange %s, error: %s" %
(virt_host, exchange_name, e))
finally:
if connected:
connection.close()
return success
class MessageQueuePipeline(object):
"""Emit processed items to a RabbitMQ exchange/queue"""
def __init__(self, host_name, port, userid, password, virtual_host,
encoder_class):
self.connection = Connection(hostname=host_name,
port=port,
userid=userid,
password=password,
virtual_host=virtual_host)
self.encoder = encoder_class()
dispatcher.connect(self.spider_opened, signals.spider_opened)
dispatcher.connect(self.spider_closed, signals.spider_closed)
@classmethod
def from_settings(cls, settings):
host_name = settings.get('BROKER_HOST')
port = settings.get('BROKER_PORT')
userid = settings.get('BROKER_USERID')
password = settings.get('BROKER_PASSWORD')
virtual_host = settings.get('BROKER_VIRTUAL_HOST')
encoder_class = settings.get('MESSAGE_Q_SERIALIZER', ScrapyJSONEncoder)
return cls(host_name, port, userid, password, virtual_host,
encoder_class)
def spider_opened(self, spider):
self.queue = self.connection.SimpleQueue(spider.name)
def spider_closed(self, spider):
self.queue.close()
self.connection.close()
def process_item(self, item, spider):
return deferToThread(self._process_item, item, spider)
def _process_item(self, item, spider):
self.queue.put(self.encoder.encode(dict(item)))
return item
def test_close_ResponseError(self):
c = Connection(transport=Transport).channel()
c.client.bgsave_raises_ResponseError = True
c.close()
def test_close_poller_not_active(self):
c = Connection(transport=Transport).channel()
cycle = c.connection.cycle
c.client.connection
c.close()
self.assertNotIn(c, cycle._channels)
class test_Redis(TestCase):
def setUp(self):
self.connection = Connection(transport=Transport)
self.exchange = Exchange('test_Redis', type='direct')
self.queue = Queue('test_Redis', self.exchange, 'test_Redis')
def tearDown(self):
self.connection.close()
def test_publish__get(self):
channel = self.connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
self.queue(channel).declare()
producer.publish({'hello': 'world'})
self.assertDictEqual(self.queue(channel).get().payload,
{'hello': 'world'})
self.assertIsNone(self.queue(channel).get())
self.assertIsNone(self.queue(channel).get())
self.assertIsNone(self.queue(channel).get())
def test_publish__consume(self):
connection = Connection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
consumer = Consumer(channel, self.queue)
producer.publish({'hello2': 'world2'})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertIn(channel, channel.connection.cycle._channels)
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
with self.assertRaises(socket.timeout):
connection.drain_events(timeout=0.01)
finally:
channel.close()
def test_purge(self):
channel = self.connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
self.queue(channel).declare()
for i in range(10):
producer.publish({'hello': 'world-%s' % (i, )})
self.assertEqual(channel._size('test_Redis'), 10)
self.assertEqual(self.queue(channel).purge(), 10)
channel.close()
def test_db_values(self):
c1 = Connection(virtual_host=1,
transport=Transport).channel()
self.assertEqual(c1.client.db, 1)
c2 = Connection(virtual_host='1',
transport=Transport).channel()
self.assertEqual(c2.client.db, 1)
c3 = Connection(virtual_host='/1',
transport=Transport).channel()
self.assertEqual(c3.client.db, 1)
with self.assertRaises(Exception):
Connection(virtual_host='/foo',
transport=Transport).channel()
def test_db_port(self):
c1 = Connection(port=None, transport=Transport).channel()
self.assertEqual(c1.client.port, Transport.default_port)
c1.close()
c2 = Connection(port=9999, transport=Transport).channel()
self.assertEqual(c2.client.port, 9999)
c2.close()
def test_close_poller_not_active(self):
c = Connection(transport=Transport).channel()
cycle = c.connection.cycle
c.client.connection
c.close()
self.assertNotIn(c, cycle._channels)
def test_close_ResponseError(self):
c = Connection(transport=Transport).channel()
c.client.bgsave_raises_ResponseError = True
c.close()
def test_close_disconnects(self):
c = Connection(transport=Transport).channel()
conn1 = c.client.connection
conn2 = c.subclient.connection
c.close()
self.assertTrue(conn1.disconnected)
self.assertTrue(conn2.disconnected)
def test_get__Empty(self):
channel = self.connection.channel()
with self.assertRaises(Empty):
channel._get('does-not-exist')
channel.close()
def test_get_client(self):
myredis, exceptions = _redis_modules()
@module_exists(myredis, exceptions)
def _do_test():
conn = Connection(transport=Transport)
chan = conn.channel()
self.assertTrue(chan.Client)
self.assertTrue(chan.ResponseError)
self.assertTrue(conn.transport.connection_errors)
self.assertTrue(conn.transport.channel_errors)
_do_test()
class RabbitMq:
"manage rabbitmq."
def __init__(self,mq_server):
""
self.connection = Connection(mq_server)
self.chan = self.connection.channel()
# 关闭
def mq_close(self):
""
self.connection.close()
self.chan.close()
# 声明queue
def queue_declare(self,**kwargs):
""
queue_name = kwargs.pop('queue_name', None)
durable_flag = kwargs.pop('durable_flag', True)
exclusive_flag = kwargs.pop('exclusive_flag', False)
auto_delete_flag = kwargs.pop('auto_delete_flag', False)
self.chan.queue_declare(queue=queue_name, durable=durable_flag, exclusive=exclusive_flag, auto_delete=auto_delete_flag)
# 声明exchange
def exchange_declare(self,**kwargs):
""
exchange_name = kwargs.pop('exchange_name', None)
mq_type = kwargs.pop('mq_type', None)
durable_flag = kwargs.pop('durable_flag', True)
auto_delete_flag = kwargs.pop('auto_delete_flag', False)
self.chan.exchange_declare(exchange=exchange_name, type=mq_type, durable=durable_flag, auto_delete=auto_delete_flag)
# 绑定queue 和 exchange
def queue_bind(self,**kwargs):
""
queue_name = kwargs.pop('queue_name', None)
exchange_name = kwargs.pop('exchange_name', None)
routing_key = kwargs.pop('routing_key', None)
self.chan.queue_bind(queue=queue_name, exchange=exchange_name, routing_key=routing_key)
# 发送消息
def mq_send(self,**kwargs):
""
msg = kwargs.pop('msg', None)
exchange_name = kwargs.pop('exchange_name', None)
routing_key = kwargs.pop('routing_key', None)
message = amqp.Message(str(msg))
self.chan.basic_publish(message,exchange=exchange_name,routing_key=routing_key)
#接收消息
def mq_receive(self,**kwargs):
""
queue_name = kwargs.pop('queue_name', None)
callback = kwargs.pop('callback', None)
self.chan.basic_consume(callback=callback,queue=queue_name,no_ack=True)
while True:
self.chan.wait()
def test_close_ResponseError(self):
c = Connection(transport=Transport).channel()
c.client.bgsave_raises_ResponseError = True
c.close()
def test_close_poller_not_active(self):
c = Connection(transport=Transport).channel()
cycle = c.connection.cycle
c.client.connection
c.close()
self.assertNotIn(c, cycle._channels)
class test_Redis(TestCase):
def setUp(self):
self.connection = Connection(transport=Transport)
self.exchange = Exchange('test_Redis', type='direct')
self.queue = Queue('test_Redis', self.exchange, 'test_Redis')
def tearDown(self):
self.connection.close()
def test_publish__get(self):
channel = self.connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
self.queue(channel).declare()
producer.publish({'hello': 'world'})
self.assertDictEqual(self.queue(channel).get().payload,
{'hello': 'world'})
self.assertIsNone(self.queue(channel).get())
self.assertIsNone(self.queue(channel).get())
self.assertIsNone(self.queue(channel).get())
def test_publish__consume(self):
connection = Connection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
consumer = Consumer(channel, self.queue)
producer.publish({'hello2': 'world2'})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertIn(channel, channel.connection.cycle._channels)
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
with self.assertRaises(socket.timeout):
connection.drain_events(timeout=0.01)
finally:
channel.close()
def test_purge(self):
channel = self.connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
self.queue(channel).declare()
for i in range(10):
producer.publish({'hello': 'world-%s' % (i, )})
self.assertEqual(channel._size('test_Redis'), 10)
self.assertEqual(self.queue(channel).purge(), 10)
channel.close()
def test_db_values(self):
c1 = Connection(virtual_host=1,
transport=Transport).channel()
self.assertEqual(c1.client.db, 1)
c2 = Connection(virtual_host='1',
transport=Transport).channel()
self.assertEqual(c2.client.db, 1)
c3 = Connection(virtual_host='/1',
transport=Transport).channel()
self.assertEqual(c3.client.db, 1)
with self.assertRaises(Exception):
Connection(virtual_host='/foo',
transport=Transport).channel()
def test_db_port(self):
c1 = Connection(port=None, transport=Transport).channel()
self.assertEqual(c1.client.port, Transport.default_port)
c1.close()
c2 = Connection(port=9999, transport=Transport).channel()
self.assertEqual(c2.client.port, 9999)
c2.close()
def test_close_poller_not_active(self):
c = Connection(transport=Transport).channel()
cycle = c.connection.cycle
c.client.connection
c.close()
self.assertNotIn(c, cycle._channels)
def test_close_ResponseError(self):
c = Connection(transport=Transport).channel()
c.client.bgsave_raises_ResponseError = True
c.close()
def test_close_disconnects(self):
c = Connection(transport=Transport).channel()
conn1 = c.client.connection
conn2 = c.subclient.connection
c.close()
self.assertTrue(conn1.disconnected)
self.assertTrue(conn2.disconnected)
def test_get__Empty(self):
channel = self.connection.channel()
with self.assertRaises(Empty):
channel._get('does-not-exist')
channel.close()
def test_get_client(self):
myredis, exceptions = _redis_modules()
@module_exists(myredis, exceptions)
def _do_test():
conn = Connection(transport=Transport)
chan = conn.channel()
self.assertTrue(chan.Client)
self.assertTrue(chan.ResponseError)
self.assertTrue(conn.transport.connection_errors)
self.assertTrue(conn.transport.channel_errors)
_do_test()
