class UriConnectionTest(unittest.TestCase):
def test_uri_connection(self):
self.connection = UriConnection(URI)
self.channel = self.connection.channel()
self.assertTrue(self.connection.is_open)
self.channel.close()
self.connection.close()
class UriConnectionTest(unittest.TestCase):
def test_uri_connection(self):
self.connection = UriConnection(URI)
self.channel = self.connection.channel()
self.assertTrue(self.connection.is_open)
self.channel.close()
self.connection.close()
def test_uri_get_ssl_validation(self):
connection = UriConnection(
'amqps://*****:*****@localhost:5672/%2F', lazy=True
)
self.assertEqual(ssl.CERT_REQUIRED,
connection._get_ssl_validation('cert_required'))
def test_uri_get_ssl_version(self):
connection = UriConnection(
'amqp://*****:*****@localhost:5672/%2F', lazy=True
)
self.assertEqual(ssl.PROTOCOL_TLSv1,
connection._get_ssl_version('protocol_tlsv1'))
def connection(self):
"""The :class:amqpstorm.Connection` for the current
proccess. This property may change without notice.
"""
with self.lock:
if self._connection is None or self._connection.is_closed:
self._connection = UriConnection(self.url)
return self._connection
def test_functional_ssl_uri_connection_with_context(self):
ssl_options = {
'context': ssl.create_default_context(cafile=CAFILE),
'server_hostname': SSL_HOST
}
self.connection = UriConnection(SSL_URI, ssl_options=ssl_options)
self.channel = self.connection.channel()
self.assertTrue(self.connection.is_open)
def test_uri_get_invalid_ssl_validation(self):
connection = \
UriConnection('amqps://*****:*****@localhost:5672/%2F', True)
self.assertEqual(ssl.CERT_NONE,
connection._get_ssl_validation('cert_test'))
self.assertIn(
"ssl_options: cert_reqs 'cert_test' not found "
"falling back to CERT_NONE.", self.get_last_log())
def test_uri_get_invalid_ssl_version(self):
connection = \
UriConnection('amqps://*****:*****@localhost:5672/%2F', True)
self.assertEqual(connection._get_ssl_version('protocol_test'),
ssl.PROTOCOL_TLSv1)
self.assertIn(
"ssl_options: ssl_version 'protocol_test' not found "
"falling back to PROTOCOL_TLSv1.", self.get_last_log())
def test_uri_get_invalid_ssl_version(self):
connection = UriConnection(
'amqps://*****:*****@localhost:5672/%2F', lazy=True
)
self.assertEqual(connection._get_ssl_version('protocol_test'),
ssl.PROTOCOL_TLSv1)
self.assertIn("ssl_options: ssl_version 'protocol_test' not found "
"falling back to PROTOCOL_TLSv1.",
self.get_last_log())
def test_uri_invalid_ssl_options(self):
connection = \
UriConnection('amqps://*****:*****@localhost:5672/%2F', True)
ssl_kwargs = {
'unit_test': ['not_required'],
}
ssl_options = connection._parse_ssl_options(ssl_kwargs)
self.assertFalse(ssl_options)
self.assertIn("invalid option: unit_test", self.get_last_log())
def test_uri_get_invalid_ssl_validation(self):
connection = UriConnection(
'amqps://*****:*****@localhost:5672/%2F', lazy=True
)
self.assertEqual(ssl.CERT_NONE,
connection._get_ssl_validation('cert_test'))
self.assertIn("ssl_options: cert_reqs 'cert_test' not found "
"falling back to CERT_NONE.",
self.get_last_log())
def test_uri_invalid_ssl_options(self):
connection = \
UriConnection('amqps://*****:*****@localhost:5672/%2F', True)
ssl_kwargs = {
'unit_test': ['not_required'],
}
ssl_options = connection._parse_ssl_options(ssl_kwargs)
self.assertFalse(ssl_options)
self.assertIn("invalid option: unit_test",
self.logging_handler.messages['warning'][0])
def test_uri_invalid_ssl_options(self):
connection = UriConnection(
'amqps://*****:*****@localhost:5672/%2F', lazy=True
)
ssl_kwargs = {
'unit_test': ['not_required'],
}
ssl_options = connection._parse_ssl_options(ssl_kwargs)
self.assertFalse(ssl_options)
self.assertIn("invalid option: unit_test",
self.get_last_log())
def test_uri_get_ssl_options(self):
connection = \
UriConnection('amqp://*****:*****@localhost:5672/%2F', True)
ssl_kwargs = {
'cert_reqs': ['cert_required'],
'ssl_version': ['protocol_tlsv1'],
'keyfile': ['file.key'],
'certfile': ['file.crt']
}
ssl_options = connection._parse_ssl_options(ssl_kwargs)
self.assertEqual(ssl_options['cert_reqs'], ssl.CERT_REQUIRED)
self.assertEqual(ssl_options['ssl_version'], ssl.PROTOCOL_TLSv1)
self.assertEqual(ssl_options['keyfile'], 'file.key')
self.assertEqual(ssl_options['certfile'], 'file.crt')
def test_uri_get_ssl_options(self):
connection = \
UriConnection('amqp://*****:*****@localhost:5672/%2F', True)
ssl_kwargs = {
'cert_reqs': ['cert_required'],
'ssl_version': ['protocol_tlsv1'],
'keyfile': ['file.key'],
'certfile': ['file.crt']
}
ssl_options = connection._parse_ssl_options(ssl_kwargs)
self.assertEqual(ssl_options['cert_reqs'], ssl.CERT_REQUIRED)
self.assertEqual(ssl_options['ssl_version'], ssl.PROTOCOL_TLSv1)
self.assertEqual(ssl_options['keyfile'], 'file.key')
self.assertEqual(ssl_options['certfile'], 'file.crt')
def test_uri_set_timeout(self):
connection = \
UriConnection('amqps://*****:*****@localhost:5672/%2F?'
'timeout=1337', True)
self.assertIsInstance(connection.parameters['timeout'], int)
self.assertEqual(connection.parameters['timeout'], 1337)
def test_functional_open_close_connection_loop(self):
self.connection = Connection(HOST, USERNAME, PASSWORD, lazy=True)
for _ in range(25):
self.connection.open()
channel = self.connection.channel()
# Make sure that it's a new channel.
self.assertEqual(int(channel), 1)
channel.queue.declare(self.queue_name)
channel.close()
# Verify that the Connection/Channel has been opened properly.
self.assertIsNotNone(self.connection._io.socket)
self.assertIsNotNone(self.connection._io.poller)
self.assertTrue(self.connection.is_open)
self.connection.close()
# Verify that the Connection has been closed properly.
self.assertTrue(self.connection.is_closed)
self.assertIsNone(self.connection._io.socket)
self.assertIsNone(self.connection._io.poller)
self.assertFalse(self.connection._io._running.is_set())
self.assertFalse(self.connection.exceptions)
def test_functional_open_close_connection_loop(self):
self.connection = Connection(HOST, USERNAME, PASSWORD, lazy=True)
for _ in range(25):
self.connection.open()
channel = self.connection.channel()
# Make sure that it's a new channel.
self.assertEqual(int(channel), 1)
channel.queue.declare(self.queue_name)
channel.close()
# Verify that the Connection/Channel has been opened properly.
self.assertIsNotNone(self.connection._io.socket)
self.assertIsNotNone(self.connection._io.poller)
self.assertTrue(self.connection.is_open)
self.connection.close()
# Verify that the Connection has been closed properly.
self.assertTrue(self.connection.is_closed)
self.assertIsNone(self.connection._io.socket)
self.assertIsNone(self.connection._io.poller)
self.assertFalse(self.connection._io._running.is_set())
self.assertFalse(self.connection.exceptions)
def test_uri_set_password(self):
connection = \
UriConnection('amqps://*****:*****@localhost:5672/%2F?'
'heartbeat=1337', True)
self.assertIsInstance(connection.parameters['password'], str)
self.assertEqual(connection.parameters['password'], 'password')
def test_uri_set_username(self):
connection = \
UriConnection('amqps://*****:*****@localhost:5672/%2F?'
'heartbeat=1337', True)
self.assertIsInstance(connection.parameters['username'], str)
self.assertEqual(connection.parameters['username'], 'username')
def test_uri_set_heartbeat(self):
connection = \
UriConnection('amqps://*****:*****@localhost:5672/%2F?'
'heartbeat=360', True)
self.assertIsInstance(connection.parameters['heartbeat'], int)
self.assertEqual(connection.parameters['heartbeat'], 360)
def test_uri_set_hostname(self):
connection = \
UriConnection('amqps://*****:*****@my-server:5672/%2F?'
'heartbeat=1337', True)
self.assertIsInstance(connection.parameters['hostname'], str)
self.assertEqual(connection.parameters['hostname'], 'my-server')
def test_uri_simple(self):
connection = \
UriConnection('amqps://localhost:5672/%2F', True)
self.assertEqual(connection.parameters['hostname'], 'localhost')
self.assertEqual(connection.parameters['username'], 'guest')
self.assertEqual(connection.parameters['password'], 'guest')
def test_uri_set_client_properties(self):
cp = {'platform': 'Atari', 'license': 'MIT'}
connection = UriConnection('amqp://*****:*****@localhost:5672/%2F',
lazy=True,
client_properties=cp)
self.assertIsInstance(connection.parameters['client_properties'], dict)
self.assertEqual(connection.parameters['client_properties'], cp)
def connect(self):
"""Create a connection.
:return:
"""
attempts = 0
while True:
attempts += 1
try:
if self._router._connection_type == "app":
request_res = requests.post(
"https://api.kervi.io/sessions",
headers={
"api-user": self._user,
"api-password": self._password,
"app_id": self._app_id,
"app_name": self._app_name,
})
print("cres", request_res.json())
connection_string = 'amqps://' + self._user + ':' + self._password + '@' + self._address + ':' + str(
self._port) + '/' + self._vhost
#print("cns", connection_string)
self._connection = UriConnection(connection_string)
#self._connection = Connection(self._address, self._user, self._password, port=self._port, vhost=self._vhost)
break
except amqpstorm.AMQPError as why:
LOGGER.exception(why)
print("why connect", why.error_code)
if why.error_code == 403 or why.error_code == 530:
print(
"Kervi.io authentication error, check configuration for the plugin kervi.plugin.routing.kervi_io"
)
break
if self._max_retries and attempts > self._max_retries:
break
time.sleep(min(attempts * 2, 30))
except KeyboardInterrupt:
break
if self._connection:
self._consumer = _MQConsumer(self)
self._consumer.start()
self._publisher = _MQPublisher(self)
self._router.on_connected()
def connect(self, force=False):
"""Connect to the Message Broker supporting AMQP(S)."""
if force:
LOG.debug("Force close the connection")
self.close()
if self.conn:
# LOG.debug("We already have a connection")
if not self.conn.is_closed:
# LOG.debug("connection not closed, returning it")
return
self.close()
if not self.connection_params:
self.fetch_args()
self.conn = UriConnection(self.connection_params,
ssl_options=self.ssl_options,
client_properties=self.client_properties,
lazy=True) # don't start it
# Retry loop
backoff = self.interval
for count in range(1, self.attempts + 1):
try:
self.conn.open()
LOG.debug("Connection successful")
return
except AMQPConnectionError as e:
self.conn.close(
) # when we can't open, we must close the unused socket
LOG.error("Opening MQ Connection retry attempt %d", count)
LOG.error('Reason %r', e)
sleep(backoff)
backoff = (2**(count // 10)) * self.interval
# from 0 to 9, sleep 1 * interval secs
# from 10 to 19, sleep 2 * interval secs
# from 20 to 29, sleep 4 * interval secs ... etc
# fail
if callable(self.on_failure):
LOG.error("Failed to open the connection")
self.on_failure()
def test_uri_default(self):
connection = \
UriConnection('amqp://*****:*****@localhost:5672/%2F', True)
self.assertEqual(connection.parameters['hostname'], 'localhost')
self.assertEqual(connection.parameters['username'], 'guest')
self.assertEqual(connection.parameters['password'], 'guest')
self.assertEqual(connection.parameters['virtual_host'], '/')
self.assertEqual(connection.parameters['port'], 5672)
self.assertEqual(connection.parameters['heartbeat'], 60)
self.assertEqual(connection.parameters['timeout'], 30)
self.assertFalse(connection.parameters['ssl'])
def test_uri_default(self):
connection = UriConnection('amqp://*****:*****@localhost:5672/%2F',
lazy=True)
self.assertEqual(connection.parameters['hostname'], 'localhost')
self.assertEqual(connection.parameters['username'], 'guest')
self.assertEqual(connection.parameters['password'], 'guest')
self.assertEqual(connection.parameters['virtual_host'],
DEFAULT_VIRTUAL_HOST)
self.assertEqual(connection.parameters['port'], 5672)
self.assertEqual(connection.parameters['heartbeat'],
DEFAULT_HEARTBEAT_INTERVAL)
self.assertEqual(connection.parameters['timeout'],
DEFAULT_SOCKET_TIMEOUT)
self.assertFalse(connection.parameters['ssl'])
def test_uri_set_ssl(self):
connection = UriConnection(
'amqps://*****:*****@localhost:5671/%2F?'
'ssl_version=protocol_tlsv1&cert_reqs=cert_required&'
'keyfile=file.key&certfile=file.crt&'
'ca_certs=travis-ci', True)
self.assertTrue(connection.parameters['ssl'])
self.assertEqual(connection.parameters['ssl_options']['ssl_version'],
ssl.PROTOCOL_TLSv1)
self.assertEqual(connection.parameters['ssl_options']['cert_reqs'],
ssl.CERT_REQUIRED)
self.assertEqual(connection.parameters['ssl_options']['keyfile'],
'file.key')
self.assertEqual(connection.parameters['ssl_options']['certfile'],
'file.crt')
self.assertEqual(connection.parameters['ssl_options']['ca_certs'],
'travis-ci')
def test_uri_get_ssl_options_new_method(self):
ssl_kwargs = {
'cert_reqs': ssl.CERT_REQUIRED,
'ssl_version': ssl.PROTOCOL_TLSv1,
'keyfile': 'file.key',
'certfile': 'file.crt'
}
connection = UriConnection(
'amqps://*****:*****@localhost:5671/%2F?'
'server_hostname=rmq.eandersson.net&certfile=file.crt',
ssl_options=ssl_kwargs,
lazy=True)
ssl_options = connection.parameters.get('ssl_options')
self.assertEqual(ssl_options['server_hostname'], 'rmq.eandersson.net')
self.assertEqual(ssl_options['cert_reqs'], ssl.CERT_REQUIRED)
self.assertEqual(ssl_options['ssl_version'], ssl.PROTOCOL_TLSv1)
self.assertEqual(ssl_options['keyfile'], 'file.key')
self.assertEqual(ssl_options['certfile'], 'file.crt')
def connect(self, amqp_uri, events_bindings: Dict[str, Callable],
exchange_name):
if exchange_name not in self._exchanges:
if amqp_uri not in self._connections:
self._connections[amqp_uri] = {
'_conn': UriConnection(amqp_uri)
}
self._connections[amqp_uri]['_ch'] = self._connections[
amqp_uri]['_conn'].channel()
channel = self._connections[amqp_uri]['_ch']
# Declare direct exchange
channel.exchange.declare(exchange=exchange_name, durable=True)
for event_name, callback in events_bindings.items():
declared_queue = channel.queue.declare(queue=event_name)
queue_name = declared_queue.get('queue')
channel.queue.bind(queue_name, exchange_name, event_name)
# Define queue callback function
channel.basic.consume(queue=queue_name, callback=callback)
self._exchanges[exchange_name] = channel
else:
raise ValueError(
'Already connected to server / exchange combination')
r.publish(token + "face", rid)
#print(token,rid)
return
while True:
result = childChannel.basic.get("tf-task", no_ack=False)
if not result:
childChannel.basic.consume(processTask, "tf-task", no_ack=False)
childChannel.start_consuming()
continue
processTask(result)
while True:
try:
mainConnection = UriConnection(config.RABBITMQ_CONFIG['uri'])
break
except:
print("Couldn't connect to rabbitMQ server. Retrying")
time.sleep(3)
continue
global mainchannel_out
mainChannel_out = mainConnection.channel()
mainChannel_in = mainConnection.channel()
def processMessage(message):
message.ack()
#print(message.body)
tensorThread(message.body) ##
def test_uri_set_virtual_host(self):
connection = \
UriConnection('amqps://*****:*****@localhost:5672/travis', True)
self.assertIsInstance(connection.parameters['virtual_host'], str)
self.assertEqual(connection.parameters['virtual_host'], 'travis')
def test_uri_set_port(self):
connection = \
UriConnection('amqps://*****:*****@localhost:1337/%2F', True)
self.assertIsInstance(connection.parameters['port'], int)
self.assertEqual(connection.parameters['port'], 1337)
class ManoBrokerConnection(object):
"""
This class encapsulates a bare RabbitMQ connection setup.
It provides helper methods to easily publish/subscribe to a given topic.
It uses the asynchronous adapter implementation of the amqpstorm library.
"""
def __init__(self, app_id, **kwargs):
"""
Initialize broker connection.
:param app_id: string that identifies application
"""
self.app_id = app_id
# fetch configuration
if "url" in kwargs:
self.rabbitmq_url = kwargs['url']
else:
self.rabbitmq_url = os.environ.get("broker_host", RABBITMQ_URL_FALLBACK)
self.rabbitmq_exchange = os.environ.get("broker_exchange", RABBITMQ_EXCHANGE_FALLBACK)
self.rabbitmq_exchange_type = "topic"
# create additional members
self._connection = None
# trigger connection setup (without blocking)
self.setup_connection()
# Threading workers
self.thrd_pool = pool.ThreadPoolExecutor(max_workers=100)
# Track the workers
self.tasks = []
def setup_connection(self):
"""
Connect to rabbit mq using self.rabbitmq_url.
"""
self._connection = UriConnection(self.rabbitmq_url)
return self._connection
def stop_connection(self):
"""
Close the connection
:return:
"""
self._connection.close()
def stop_threads(self):
"""
Stop all the threads that are consuming messages
"""
for task in self.tasks:
task.cancel()
def publish(self, topic, message, properties=None):
"""
This method provides basic topic-based message publishing.
:param topic: topic the message is published to
:param message: the message (JSON/YAML/STRING)
:param properties: custom properties for the message (as dict)
:return:
"""
# create a new channel
with self._connection.channel() as channel:
# declare the exchange to be used
channel.exchange.declare(self.rabbitmq_exchange, exchange_type=self.rabbitmq_exchange_type)
# update the default properties with custom ones from the properties argument
if properties is None:
properties = dict()
default_properties = {
"app_id": self.app_id,
"content_type": "application/json",
"correlation_id": None,
"reply_to": None,
"headers": dict()
}
default_properties.update(properties)
# fix properties (amqpstorm does not like None values):
for k, v in default_properties.items():
default_properties[k] = "" if v is None else v
if "headers" in default_properties:
for k, v in default_properties["headers"].items():
default_properties["headers"][k] = "" if v is None else v
# publish the message
channel.basic.publish(body=message,
routing_key=topic,
exchange=self.rabbitmq_exchange,
properties=default_properties)
LOG.debug("PUBLISHED to %r: %r", topic, message)
def subscribe(self, cbf, topic, subscription_queue=None):
"""
Implements basic subscribe functionality.
Starts a new thread for each subscription in which messages are consumed and the callback functions
are called.
:param cbf: callback function cbf(channel, method, properties, body)
:param topic: topic to subscribe to
:return:
"""
def _wrapper_cbf(msg):
"""
This internal cbf translates amqpstorm message arguments
to pika's legacy cbf argument format.
:param msg: amqp message
:return:
"""
# translate msg properties
ch = msg.channel
body = msg.body
method = type('method', (object,), msg.method)
# ensure that we have a header field
if "headers" not in msg.properties:
msg.properties["headers"] = dict()
# make emtpy strings to None to be compatible
for k, v in msg.properties.items():
msg.properties[k] = None if v == "" else v
properties = type('properties', (object,), msg.properties)
# call cbf of subscription
try:
cbf(ch, method, properties, body)
except BaseException as e:
LOG.error("Error in subscription thread: " + str(''.join(traceback.format_tb(e.__traceback__))))
# ack the message to let broker know that message was delivered
msg.ack()
def connection_thread():
"""
Each subscription consumes messages in its own thread.
:return:
"""
with self._connection.channel() as channel:
# declare exchange for this channes
channel.exchange.declare(exchange=self.rabbitmq_exchange, exchange_type=self.rabbitmq_exchange_type)
# create queue for subscription
q = channel.queue
q.declare(subscription_queue)
# bind queue to given topic
q.bind(queue=subscription_queue, routing_key=topic, exchange=self.rabbitmq_exchange)
# recommended qos setting
channel.basic.qos(100)
# setup consumer (use queue name as tag)
channel.basic.consume(_wrapper_cbf, subscription_queue, consumer_tag=subscription_queue, no_ack=False)
try:
# start consuming messages.
channel.start_consuming(to_tuple=False)
except BaseException as e:
LOG.error("Error in subscription thread: " + str(''.join(traceback.format_tb(e.__traceback__))))
channel.close()
# Attention: We crate an individual queue for each subscription to allow multiple subscriptions
# to the same topic.
if subscription_queue is None:
queue_uuid = str(uuid.uuid4())
subscription_queue = "%s.%s.%s" % ("q", topic, queue_uuid)
# each subscriber is an own thread
LOG.debug("start new thread to consume " + str(subscription_queue))
task = self.thrd_pool.submit(connection_thread)
task.add_done_callback(self.done_with_task)
self.tasks.append(task)
#Make sure that consuming has started, before method finishes.
time.sleep(0.1)
LOG.debug("SUBSCRIBED to %r", topic)
return subscription_queue
def done_with_task(self, f):
"""
def test_uri_get_invalid_ssl_validation(self):
connection = \
UriConnection('amqp://*****:*****@localhost:5672/%2F', True)
self.assertEqual(ssl.CERT_NONE,
connection._get_ssl_validation('cert_test'))
def test_uri_ssl(self):
connection = \
UriConnection('amqps://*****:*****@localhost:5672/%2F', True)
self.assertTrue(connection.parameters['ssl'])
def test_uri_get_invalid_ssl_version(self):
connection = \
UriConnection('amqp://*****:*****@localhost:5672/%2F', True)
self.assertEqual(connection._get_ssl_version('protocol_test'),
ssl.PROTOCOL_TLSv1)
def test_uri_get_ssl_validation(self):
connection = \
UriConnection('amqps://*****:*****@localhost:5672/%2F', True)
self.assertEqual(ssl.CERT_REQUIRED,
connection._get_ssl_validation('cert_required'))
def test_functional_uri_connection(self):
self.connection = UriConnection(URI)
self.channel = self.connection.channel()
self.assertTrue(self.connection.is_open)
def setup_connection(self):
"""
Connect to rabbit mq using self.rabbitmq_url.
"""
self._connection = UriConnection(self.rabbitmq_url)
return self._connection
class ManoBrokerConnection(object):
"""
This class encapsulates a bare RabbitMQ connection setup.
It provides helper methods to easily publish/subscribe to a given topic.
It uses the asynchronous adapter implementation of the amqpstorm library.
"""
def __init__(self, app_id, **kwargs):
"""
Initialize broker connection.
:param app_id: string that identifies application
"""
self.app_id = app_id
# fetch configuration
if "url" in kwargs:
self.rabbitmq_url = kwargs['url']
else:
self.rabbitmq_url = os.environ.get("broker_host",
RABBITMQ_URL_FALLBACK)
self.rabbitmq_exchange = os.environ.get("broker_exchange",
RABBITMQ_EXCHANGE_FALLBACK)
self.rabbitmq_exchange_type = "topic"
# create additional members
self._connection = None
# trigger connection setup (without blocking)
self.setup_connection()
# Threading workers
self.thrd_pool = pool.ThreadPoolExecutor(max_workers=100)
# Track the workers
self.tasks = []
def setup_connection(self):
"""
Connect to rabbit mq using self.rabbitmq_url.
"""
self._connection = UriConnection(self.rabbitmq_url)
return self._connection
def stop_connection(self):
"""
Close the connection
:return:
"""
self._connection.close()
def stop_threads(self):
"""
Stop all the threads that are consuming messages
"""
for task in self.tasks:
task.cancel()
def publish(self, topic, message, properties=None):
"""
This method provides basic topic-based message publishing.
:param topic: topic the message is published to
:param message: the message (JSON/YAML/STRING)
:param properties: custom properties for the message (as dict)
:return:
"""
# create a new channel
with self._connection.channel() as channel:
# declare the exchange to be used
channel.exchange.declare(self.rabbitmq_exchange,
exchange_type=self.rabbitmq_exchange_type)
# update the default properties with custom ones from the properties argument
if properties is None:
properties = dict()
default_properties = {
"app_id": self.app_id,
"content_type": "application/json",
"correlation_id": None,
"reply_to": None,
"headers": dict()
}
default_properties.update(properties)
# fix properties (amqpstorm does not like None values):
for k, v in default_properties.items():
default_properties[k] = "" if v is None else v
if "headers" in default_properties:
for k, v in default_properties["headers"].items():
default_properties["headers"][k] = "" if v is None else v
# publish the message
channel.basic.publish(body=message,
routing_key=topic,
exchange=self.rabbitmq_exchange,
properties=default_properties)
LOG.debug("PUBLISHED to %r: %r", topic, message)
def subscribe(self, cbf, topic, subscription_queue=None):
"""
Implements basic subscribe functionality.
Starts a new thread for each subscription in which messages are consumed and the callback functions
are called.
:param cbf: callback function cbf(channel, method, properties, body)
:param topic: topic to subscribe to
:return:
"""
def _wrapper_cbf(msg):
"""
This internal cbf translates amqpstorm message arguments
to pika's legacy cbf argument format.
:param msg: amqp message
:return:
"""
# translate msg properties
ch = msg.channel
body = msg.body
method = type('method', (object, ), msg.method)
# ensure that we have a header field
if "headers" not in msg.properties:
msg.properties["headers"] = dict()
# make emtpy strings to None to be compatible
for k, v in msg.properties.items():
msg.properties[k] = None if v == "" else v
properties = type('properties', (object, ), msg.properties)
# call cbf of subscription
try:
cbf(ch, method, properties, body)
except BaseException as e:
LOG.error("Error in subscription thread: " + str(e) + '\n' +
str(''.join(traceback.format_tb(e.__traceback__))))
# ack the message to let broker know that message was delivered
msg.ack()
def connection_thread():
"""
Each subscription consumes messages in its own thread.
:return:
"""
with self._connection.channel() as channel:
# declare exchange for this channes
channel.exchange.declare(
exchange=self.rabbitmq_exchange,
exchange_type=self.rabbitmq_exchange_type)
# create queue for subscription
q = channel.queue
q.declare(subscription_queue)
# bind queue to given topic
q.bind(queue=subscription_queue,
routing_key=topic,
exchange=self.rabbitmq_exchange)
# recommended qos setting
channel.basic.qos(100)
# setup consumer (use queue name as tag)
channel.basic.consume(_wrapper_cbf,
subscription_queue,
consumer_tag=subscription_queue,
no_ack=False)
try:
# start consuming messages.
channel.start_consuming(to_tuple=False)
except BaseException as e:
LOG.error(
"Error in subscription thread: " +
str(''.join(traceback.format_tb(e.__traceback__))))
channel.close()
# Attention: We crate an individual queue for each subscription to allow multiple subscriptions
# to the same topic.
if subscription_queue is None:
queue_uuid = str(uuid.uuid4())
subscription_queue = "%s.%s.%s" % ("q", topic, queue_uuid)
# each subscriber is an own thread
LOG.debug("start new thread to consume " + str(subscription_queue))
task = self.thrd_pool.submit(connection_thread)
task.add_done_callback(self.done_with_task)
self.tasks.append(task)
#Make sure that consuming has started, before method finishes.
time.sleep(0.1)
LOG.debug("SUBSCRIBED to %r", topic)
return subscription_queue
def done_with_task(self, f):
"""
def setup_connection(self):
"""
Connect to rabbit mq using self.rabbitmq_url.
"""
self._connection = UriConnection(self.rabbitmq_url)
return self._connection
class ReliabilityFunctionalTests(TestFunctionalFramework):
@setup(new_connection=False, queue=True)
def test_functional_open_new_connection_loop(self):
for _ in range(25):
self.connection = self.connection = Connection(HOST, USERNAME,
PASSWORD)
self.channel = self.connection.channel()
# Make sure that it's a new channel.
self.assertEqual(int(self.channel), 1)
self.channel.queue.declare(self.queue_name)
# Verify that the Connection/Channel has been opened properly.
self.assertIsNotNone(self.connection._io.socket)
self.assertIsNotNone(self.connection._io.poller)
self.assertTrue(self.connection.is_open)
self.channel.close()
self.connection.close()
# Verify that the Connection has been closed properly.
self.assertTrue(self.connection.is_closed)
self.assertIsNone(self.connection._io.socket)
self.assertIsNone(self.connection._io.poller)
self.assertFalse(self.connection._io._running.is_set())
self.assertFalse(self.connection.exceptions)
@setup(new_connection=False, queue=True)
def test_functional_open_close_connection_loop(self):
self.connection = Connection(HOST, USERNAME, PASSWORD, lazy=True)
for _ in range(25):
self.connection.open()
channel = self.connection.channel()
# Make sure that it's a new channel.
self.assertEqual(int(channel), 1)
channel.queue.declare(self.queue_name)
channel.close()
# Verify that the Connection/Channel has been opened properly.
self.assertIsNotNone(self.connection._io.socket)
self.assertIsNotNone(self.connection._io.poller)
self.assertTrue(self.connection.is_open)
self.connection.close()
# Verify that the Connection has been closed properly.
self.assertTrue(self.connection.is_closed)
self.assertIsNone(self.connection._io.socket)
self.assertIsNone(self.connection._io.poller)
self.assertFalse(self.connection._io._running.is_set())
self.assertFalse(self.connection.exceptions)
@setup(new_connection=True, new_channel=False, queue=True)
def test_functional_close_gracefully_after_publish_mandatory_fails(self):
for index in range(3):
channel = self.connection.channel()
# Try to publish 25 bad messages.
for _ in range(25):
try:
channel.basic.publish('', self.queue_name, '', None, True,
False)
except AMQPMessageError:
pass
# Sleep for 0.1s to make sure RabbitMQ has time to catch up.
time.sleep(0.1)
self.assertTrue(channel.exceptions)
channel.close()
@setup(new_connection=False, queue=True)
def test_functional_open_close_channel_loop(self):
self.connection = self.connection = Connection(HOST, USERNAME,
PASSWORD)
for _ in range(25):
channel = self.connection.channel()
# Verify that the Channel has been opened properly.
self.assertTrue(self.connection.is_open)
self.assertTrue(channel.is_open)
# Channel id should be staying at 1.
self.assertEqual(int(channel), 1)
channel.close()
# Verify that theChannel has been closed properly.
self.assertTrue(self.connection.is_open)
self.assertTrue(channel.is_closed)
@setup(new_connection=False, queue=True)
def test_functional_open_multiple_channels(self):
self.connection = self.connection = Connection(HOST, USERNAME,
PASSWORD, lazy=True)
for _ in range(5):
channels = []
self.connection.open()
for index in range(10):
channel = self.connection.channel()
channels.append(channel)
# Verify that the Channel has been opened properly.
self.assertTrue(channel.is_open)
self.assertEqual(int(channel), len(channels))
self.connection.close()
@setup(new_connection=False, queue=False)
def test_functional_close_performance(self):
"""Make sure closing a connection never takes longer than ~1 seconds.
:return:
"""
for _ in range(100):
self.connection = self.connection = Connection(HOST, USERNAME,
PASSWORD)
start_time = time.time()
self.connection.close()
self.assertLess(time.time() - start_time, 3)
@setup(new_connection=False)
def test_functional_uri_connection(self):
self.connection = UriConnection(URI)
self.channel = self.connection.channel()
self.assertTrue(self.connection.is_open)
def test_functional_ssl_connection_without_ssl(self):
restore_func = sys.modules['ssl']
try:
sys.modules['ssl'] = None
imp.reload(compatibility)
self.assertIsNone(compatibility.ssl)
self.assertRaisesRegexp(
AMQPConnectionError,
'Python not compiled with support for TLSv1 or higher',
Connection, HOST, USERNAME, PASSWORD, ssl=True
)
finally:
sys.modules['ssl'] = restore_func
imp.reload(compatibility)
