def init_pool(self):
"""Set up aconnection and channels if the connection is closed/doesn't exist.
Does nothing if the connection is already up.
"""
if self._connection and self._connection.is_open:
return
elif self._connection and self._connection.is_opening:
while self._connection.is_opening:
sleep(ChannelsManager.IDLE)
return
else:
# create/recreate the connection and overwrite the channels
self._connection = Connection(
ESHU_CONFIG['ADDRESS'],
ESHU_CONFIG['USER'],
ESHU_CONFIG['PASSWORD'],
virtual_host=ESHU_CONFIG['VIRTUAL_HOST'],
heartbeat=ESHU_CONFIG['HEARTBEAT'])
# clear previous channels if they exist
for channel in self._channels:
channel.close()
# create new channels
self._channels = []
for i in range(CHANNEL_POOL_SIZE):
self._channels.append(Channel(self._connection, i))
def test_functional_open_close_connection_loop(self):
self.connection = Connection(HOST, USERNAME, PASSWORD, lazy=True)
for _ in range(25):
self.connection.open()
channel = self.connection.channel()
# Make sure that it's a new channel.
self.assertEqual(int(channel), 1)
channel.queue.declare(self.queue_name)
channel.close()
# Verify that the Connection/Channel has been opened properly.
self.assertIsNotNone(self.connection._io.socket)
self.assertIsNotNone(self.connection._io.poller)
self.assertTrue(self.connection.is_open)
self.connection.close()
# Verify that the Connection has been closed properly.
self.assertTrue(self.connection.is_closed)
self.assertIsNone(self.connection._io.socket)
self.assertIsNone(self.connection._io.poller)
self.assertFalse(self.connection._io._running.is_set())
self.assertFalse(self.connection.exceptions)
def create_connection(self) -> None:
"""Create a connection.
:return:
"""
attempts = 0
while True:
attempts += 1
try:
self.connection = Connection(
self.hostname,
self.username,
self.password,
port=self.port,
ssl=self.ssl,
ssl_options=self.ssl_context,
virtual_host=self.vhost,
)
LOG.info("Established connection with AMQP server {0}".format(
self.connection))
break
except AMQPError as error:
LOG.error("Something went wrong: {0}".format(error))
if self.max_retries and attempts > self.max_retries:
break
time.sleep(min(attempts * 2, 30))
except KeyboardInterrupt:
break
def _connect(
host: str,
username: str,
password: str,
port: int = 5672,
ssl: bool = False,
ssl_options: SSLOptions = None,
virtual_host: str = None
) -> Connection:
"""
Connects to an amqp server
"""
LOGGER.debug('opening new BlockingConnection to host %s' % host)
options = {
'hostname': host,
'username': username,
'password': password,
'port': port,
'heartbeat': 5,
'timeout': 5,
}
if ssl:
options['ssl'] = ssl
options['ssl_options'] = ssl_options
if virtual_host:
options['virtual_host'] = virtual_host
connection = Connection(**options)
LOGGER.debug('opened new BlockingConnection to host %s' % host)
return connection
def test_connection_invalid_timeout_on_channel(self):
connection = Connection('127.0.0.1', 'guest', 'guest', timeout=1,
lazy=True)
self.assertRaisesRegexp(AMQPInvalidArgument,
'rpc_timeout should be an integer',
connection.channel, None)
def test_functional_ssl_open_close_channel_loop(self):
ssl_options = {
'context': ssl.create_default_context(cafile=CAFILE),
'server_hostname': SSL_HOST
}
self.connection = self.connection = Connection(SSL_HOST,
USERNAME,
PASSWORD,
port=5671,
ssl=True,
ssl_options=ssl_options)
for _ in range(25):
channel = self.connection.channel()
# Verify that the Channel has been opened properly.
self.assertTrue(self.connection.is_open)
self.assertTrue(channel.is_open)
# Channel id should be staying at 1.
self.assertEqual(int(channel), 1)
channel.close()
# Verify that theChannel has been closed properly.
self.assertTrue(self.connection.is_open)
self.assertTrue(channel.is_closed)
def test_functional_consume_with_ssl_until_empty(self):
ssl_options = {
'context': ssl.create_default_context(cafile=CAFILE),
'server_hostname': SSL_HOST
}
self.connection = self.connection = Connection(SSL_HOST,
USERNAME,
PASSWORD,
port=5671,
ssl=True,
ssl_options=ssl_options)
self.channel = self.connection.channel()
self.channel.queue.declare(self.queue_name)
self.channel.confirm_deliveries()
self.publish_messages()
channel = self.connection.channel()
channel.basic.consume(queue=self.queue_name, no_ack=False)
message_count = 0
for message in channel.build_inbound_messages(break_on_empty=True):
message_count += 1
message.ack()
result = channel.queue.declare(self.queue_name, passive=True)
self.assertEqual(result['message_count'], 0)
self.assertEqual(message_count, self.messages_to_send,
'not all messages consumed')
channel.close()
def test_functional_ssl_open_close_connection_loop(self):
ssl_options = {
'context': ssl.create_default_context(cafile=CAFILE),
'server_hostname': SSL_HOST
}
self.connection = self.connection = Connection(
SSL_HOST, USERNAME, PASSWORD, port=5671, ssl=True,
ssl_options=ssl_options, timeout=1, lazy=True)
for _ in range(5):
self.connection.open()
channel = self.connection.channel()
# Make sure that it's a new channel.
self.assertEqual(int(channel), 1)
channel.queue.declare(self.queue_name)
channel.close()
# Verify that the Connection/Channel has been opened properly.
self.assertIsNotNone(self.connection._io.socket)
self.assertIsNotNone(self.connection._io.poller)
self.assertTrue(self.connection.is_open)
self.connection.close()
# Verify that the Connection has been closed properly.
self.assertTrue(self.connection.is_closed)
self.assertIsNone(self.connection._io.socket)
self.assertIsNone(self.connection._io.poller)
self.assertFalse(self.connection._io._running.is_set())
self.assertFalse(self.connection.exceptions)
def test_connection_with_statement_when_failing(self):
try:
with Connection('127.0.0.1', 'guest', 'guest', lazy=True) as con:
con.exceptions.append(AMQPConnectionError('error'))
con.check_for_errors()
except AMQPConnectionError as why:
self.assertIsInstance(why, AMQPConnectionError)
def init(Rendering):
global rendering
rendering = Rendering
# Save the queue
global queue
queue = queue.Queue()
# Create a local messaging connection
Config = configparser.ConfigParser()
Config.read('./config/mirror_config.ini')
connection = Connection(Config.get('General', 'messaging_ip'), 'guest',
'guest')
global __channel
__channel = connection.channel()
# Create an exchange for the mirror-messages - type is direct so we can distinguish the different messages
__channel.exchange.declare(exchange='from-mirror', exchange_type='direct')
__channel.exchange.declare(exchange='from-server', exchange_type='direct')
# Declare a queue to be used (random name will be used)
result = __channel.queue.declare(exclusive=True)
queue_name = result['queue']
# Listen to server messages
for msg_keys in MSG_TO_MIRROR_KEYS:
__channel.queue.bind(exchange='from-server',
queue=queue_name,
routing_key=msg_keys.name)
__channel.basic.consume(consume_server_message,
queue=queue_name,
no_ack=True)
def test_connection_cleanup_one_channel(self):
connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
connection._channels[1] = Channel(1, connection, 0.1)
connection._cleanup_channel(1)
self.assertFalse(connection._channels)
def connect():
module.connection = Connection(current_app.config['AMQP_HOST'],
current_app.config['AMQP_USER'],
current_app.config['AMQP_PASS'],
kwargs={'heartbeat': 600})
module.channel = connection.channel()
channel.confirm_deliveries()
def test_connection_close_when_already_closed(self):
connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
connection.set_state(connection.OPEN)
io = IO(connection.parameters, [])
io.socket = Mock(name='socket', spec=socket.socket)
connection._io = io
connection.set_state(connection.CLOSED)
# Create some fake channels.
for index in range(10):
connection._channels[index + 1] = Channel(
index + 1, connection, 360)
def state_set(state):
self.assertEqual(state, connection.CLOSED)
connection.set_state = state_set
self.assertTrue(connection.is_closed)
connection.close()
# Make sure all the fake channels were closed as well.
for index in range(10):
self.assertNotIn(index + 1, connection._channels)
self.assertFalse(connection._channels)
self.assertTrue(connection.is_closed)
def test_connection_get_first_channel_id(self):
connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
self.assertEqual(connection._last_channel_id, None)
self.assertEqual(
connection._get_next_available_channel_id(), 1
)
self.assertEqual(connection._last_channel_id, 1)
def test_connection_close_handles_raise_on_write(self):
connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
connection.set_state(connection.OPEN)
io = IO(connection.parameters, [])
io.socket = Mock(name='socket', spec=socket.socket)
connection._io = io
# Create some fake channels.
for index in range(10):
connection._channels[index + 1] = Channel(
index + 1, connection, 360)
def raise_on_write(_):
raise AMQPConnectionError('travis-ci')
connection._channel0._write_frame = raise_on_write
self.assertFalse(connection.is_closed)
connection.close()
# Make sure all the fake channels were closed as well.
for index in range(10):
self.assertNotIn(index + 1, connection._channels)
self.assertFalse(connection._channels)
self.assertTrue(connection.is_closed)
def test_connection_handle_value_error(self):
"""This test covers an unlikely issue triggered by network corruption.
pamqp.decode._embedded_value raises:
ValueError: Unknown type: b'\x13'
The goal here is not to fix issues caused by network corruption,
but rather to make sure that the exceptions raised when
connections do fail are always predictable.
Fail fast and reliably!
:return:
"""
connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
def throw_error(_):
raise ValueError("Unknown type: b'\x13'")
restore_func = pamqp_frame.unmarshal
try:
pamqp_frame.unmarshal = throw_error
result = connection._handle_amqp_frame('travis-ci')
self.assertEqual(result[0], 'travis-ci')
self.assertIsNone(result[1])
self.assertIsNone(result[2])
finally:
pamqp_frame.unmarshal = restore_func
self.assertEqual(self.get_last_log(),
"Unknown type: b'\x13'")
def test_connection_close(self):
connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
connection.set_state(connection.OPEN)
io = IO(connection.parameters, [])
io.socket = Mock(name='socket', spec=socket.socket)
connection._io = io
# Create some fake channels.
for index in range(10):
connection._channels[index + 1] = Channel(
index + 1, connection, 360)
def on_write(frame_out):
self.assertIsInstance(frame_out, specification.Connection.Close)
connection._channel0._close_connection_ok()
connection._channel0._write_frame = on_write
self.assertFalse(connection.is_closed)
connection.close()
# Make sure all the fake channels were closed as well.
for index in range(10):
self.assertNotIn(index + 1, connection._channels)
self.assertTrue(connection.is_closed)
def test_connection_handle_amqp_frame_none_returns_none(self):
connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
result = connection._handle_amqp_frame('')
self.assertEqual(result[0], '')
self.assertIsNone(result[1])
self.assertIsNone(result[2])
def test_connection_handle_unicode_error(self):
"""This test covers an unlikely issue triggered by network corruption.
pamqp.decode._maybe_utf8 raises:
UnicodeDecodeError: 'utf8' codec can't
decode byte 0xc5 in position 1: invalid continuation byte
The goal here is not to fix issues caused by network corruption,
but rather to make sure that the exceptions raised when
connections do fail are always predictable.
Fail fast and reliably!
:return:
"""
connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
def throw_error(_):
raise UnicodeDecodeError(str(), bytes(), 1, 1, str())
restore_func = pamqp_frame.unmarshal
try:
pamqp_frame.unmarshal = throw_error
result = connection._handle_amqp_frame('travis-ci')
self.assertEqual(result[0], 'travis-ci')
self.assertIsNone(result[1])
self.assertIsNone(result[2])
finally:
pamqp_frame.unmarshal = restore_func
self.assertEqual(self.get_last_log(),
"'' codec can't decode bytes in position 1-0: ")
def test_functional_publish_1k_with_ssl(self):
ssl_options = {
'context': ssl.create_default_context(cafile=CAFILE),
'server_hostname': SSL_HOST
}
self.connection = self.connection = Connection(
SSL_HOST, USERNAME, PASSWORD, port=5671, ssl=True,
ssl_options=ssl_options)
self.channel = self.connection.channel()
self.channel.queue.declare(self.queue_name)
publish_thread = threading.Thread(target=self.publish_messages, )
publish_thread.daemon = True
publish_thread.start()
for _ in range(4):
consumer_thread = threading.Thread(target=self.consume_messages, )
consumer_thread.daemon = True
consumer_thread.start()
start_time = time.time()
while self.messages_consumed != self.messages_to_send:
if time.time() - start_time >= 60:
break
time.sleep(0.1)
for channel in list(self.connection.channels.values()):
channel.stop_consuming()
channel.close()
self.assertEqual(self.messages_consumed, self.messages_to_send,
'test took too long')
def publish_messages():
with Connection('127.0.0.1', 'guest', 'guest') as connection:
with connection.channel() as channel:
# Declare the Queue, 'simple_queue'.
channel.queue.declare('simple_queue')
# Message Properties.
properties = {
'content_type': 'text/plain',
'headers': {
'key': 'value'
}
}
# Enable server local transactions on channel.
channel.tx.select()
# Create the message.
message = Message.create(channel, 'Hello World!', properties)
# Publish the message to a queue called, 'simple_queue'.
message.publish('simple_queue')
# Commit the message(s).
channel.tx.commit()
# Alternatively rollback the message(s).
# channel.tx.rollback()
# You can also use the context manager.
with channel.tx:
message.publish('simple_queue')
def test_connection_cleanup_channel_does_not_exist(self):
connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
connection._channels[1] = Channel(1, connection, 0.1)
connection._cleanup_channel(2)
self.assertEqual(len(connection._channels), 1)
def test_api_connection_close(self):
reason = 'travis-ci'
api = ManagementApi(HTTP_URL, USERNAME, PASSWORD, timeout=1)
self.assertEqual(len(api.connection.list()), 0,
'found an open connection, test will fail')
connection = Connection(HOST, USERNAME, PASSWORD, timeout=1)
connections = retry_function_wrapper(api.connection.list)
self.assertIsNotNone(connections)
self.assertEqual(len(connections), 1)
for conn in api.connection.list():
self.assertEqual(api.connection.close(conn['name'], reason=reason),
None)
time.sleep(1)
self.assertRaisesRegexp(
AMQPConnectionError, 'Connection was closed by remote server: '
'CONNECTION_FORCED - %s' % reason, connection.check_for_errors)
self.assertEqual(len(api.connection.list()), 0)
connection.close()
def start_consumer():
ssl_options = {
'context': ssl.create_default_context(cafile='cacert.pem'),
'server_hostname': 'rmq.eandersson.net'
}
with Connection('rmq.eandersson.net', 'guest', 'guest', port=5671,
ssl=True, ssl_options=ssl_options) as connection:
with connection.channel() as channel:
# Declare the Queue, 'simple_queue'.
channel.queue.declare('simple_queue')
# Set QoS to 100.
# This will limit the consumer to only prefetch a 100 messages.
# This is a recommended setting, as it prevents the
# consumer from keeping all of the messages in a queue to itself.
channel.basic.qos(100)
# Start consuming the queue 'simple_queue' using the callback
# 'on_message' and last require the message to be acknowledged.
channel.basic.consume(on_message, 'simple_queue', no_ack=False)
try:
# Start consuming messages.
channel.start_consuming()
except KeyboardInterrupt:
channel.close()
def test_connection_closed_on_exception(self):
connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
connection.set_state(connection.OPEN)
connection.exceptions.append(AMQPConnectionError('travis-ci'))
self.assertTrue(connection.is_open)
self.assertRaises(AMQPConnectionError, connection.check_for_errors)
self.assertTrue(connection.is_closed)
def consumer():
with Connection('127.0.0.1', 'guest', 'guest') as connection:
with connection.channel() as channel:
channel.queue.declare('simple_queue')
channel.basic.consume(queue='simple_queue', no_ack=False)
for message in channel.build_inbound_messages():
print(message.body)
message.ack()
def connect(self):
self.connection = Connection(
'localhost', 'guest', 'guest',
kwargs={'heartbeat': 600}
)
self.channel = self.connection.channel()
self.channel.confirm_deliveries()
def test_connection_get_next_channel_id(self):
connection = Connection('127.0.0.1',
'guest',
'guest',
timeout=0.1,
lazy=True)
connection._channels[1] = None
self.assertEqual(connection._get_next_available_channel_id(), 2)
def test_connection_send_handshake(self):
connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
def on_write_to_socket(message):
self.assertEqual(message, b'AMQP\x00\x00\t\x01')
connection._io.write_to_socket = on_write_to_socket
self.assertIsNone(connection._send_handshake())
def test_connection_fileno_property(self):
connection = Connection('127.0.0.1', 'guest', 'guest', lazy=True)
connection.set_state(connection.OPENING)
io = IO(connection.parameters, [])
io.socket = Mock(name='socket', spec=socket.socket)
connection._io = io
io.socket.fileno.return_value = 5
self.assertEqual(connection.fileno, 5)
