def connection_thread(url, results, hide_password=False):
from oslo_config import cfg
from oslo_messaging.transport import TransportURL
from pika import exceptions as pika_exceptions
from pika import URLParameters as PikaUrlParameters
from pika import BlockingConnection as PikaBlockingConnection
try:
parsed_url = TransportURL.parse(cfg.CONF, url)
if hide_password:
url = re.sub(':+[^:@]+@', ':******@', url)
except Exception as e:
results.append({'url': url, 'exception': e})
else:
test_url, driver = parse_test_url(parsed_url)
try:
if driver == 'kombu':
connection = Connection(test_url)
connection.connect()
connection.close()
elif driver == 'pika':
params = PikaUrlParameters(test_url)
params.socket_timeout = 5
conn = PikaBlockingConnection(params)
conn.close()
except (OSError, pika_exceptions.ConnectionClosed):
results.append({'url': url, 'exception': _('Url not reachable')})
except (AccessRefused, pika_exceptions.ProbableAuthenticationError):
results.append({
'url': url,
'exception': _('Credentials incorrect')
})
except Exception as e:
results.append({'url': url, 'exception': force_text(e)})
else:
results.append({'url': url})
def _connect(self):
try:
logger.info("attempt to open connection",
server="primary",
category="rabbitmq")
return BlockingConnection(URLParameters(self.rabbitmq_url))
except AMQPError as e:
logger.error(
"unable to open connection",
exc_info=e,
server="primary",
category="rabbitmq",
)
try:
logger.info(
"attempt to open connection",
server="secondary",
category="rabbitmq",
)
return BlockingConnection(
URLParameters(self.rabbitmq_secondary_url))
except AMQPError as err:
logger.error(
"unable to open connection",
exc_info=e,
server="secondary",
category="rabbitmq",
)
raise err
def start(self):
"""Start worker."""
connected = False
# Define connection
while (not connected):
try:
self._creds = self._amqp_url.split('amqp://')[1].split("@")[0]
parameters = URLParameters(self._amqp_url)
self._host = parameters.host
self._port = "15672"
self._vhost = parameters.virtual_host
self._blocking_connection = BlockingConnection(parameters)
self._connection = SelectConnection(
URLParameters(self._amqp_url),
on_open_callback=self.on_open_connection)
self._connection.ioloop.start()
connected = True
# Catch a Keyboard Interrupt to make sure that the connection is closed cleanly
except KeyboardInterrupt:
# Gracefully close the connection
self._connection.close()
# Start the IOLoop again so Pika can communicate, it will stop on its own when the connection is closed
self._connection.ioloop.start()
except:
if (self._debug):
print(
" [!] RabbitMQ Host Unrecheable. Reconnecting in {} seconds..."
.format(self._reconection_time))
time.sleep(self._reconection_time)
def create_connection(self):
url_params = self._config.MQ_URI
future = Future()
TornadoConnection(URLParameters(url_params),
partial(self.on_connection_open, future),
partial(self.on_open_error, future))
return future
def decrypt_and_write():
with open(settings.SDX_SEFT_CONSUMER_KEYS_FILE) as file:
keys = yaml.safe_load(file)
key_store = KeyStore(keys)
connection = BlockingConnection(URLParameters(settings.RABBIT_URL))
channel = connection.channel()
method, properties, body = channel.basic_get(
settings.RABBIT_QUARANTINE_QUEUE)
if method:
logger.info("Recovered quarantine message",
body=body,
headers=properties.headers)
try:
decrypted_message = decrypt(body.decode("utf-8"), key_store,
KEY_PURPOSE_CONSUMER)
payload = SeftConsumer.extract_file(decrypted_message,
properties.headers['tx_id'])
with open('/tmp/{}'.format(payload.file_name),
'wb') as recovered_file:
recovered_file.write(payload.decoded_contents)
channel.basic_ack(method.delivery_tag)
logger.info("Message ACK")
except (InvalidTokenException, ValueError):
logger.exception("Bad decrypt")
channel.basic_nack(method.delivery_tag)
logger.info("Nacking message")
except Exception:
logger.exception("Failed to process")
channel.basic_nack(method.delivery_tag)
logger.info("Nacking message")
else:
logger.info('No message found on quarantine queue')
def __init__(self, rabbitmq_url, io_loop=None):
"""
A Rabbitmq client, using tornado to complete asynchronous invoking.
It is an `everything-in-one` RabbitMQ client, including following interfaces:
- establish connection
- publish
- receive
- rpc
details:
1. establish connection
Every instance of `TornadoAdapter` has two rabbitmq connections. one for publish and the other for consumer.
2. publish
Publishing connection creates a channel when invoke publish method. After publishing message, it closes channel.
3. receive
It receives message from queue and process message. and publish message if necessary.
4. rpc
It invoke rpc call and wait result asynchronously. Of course, it supports timeout.
:param rabbitmq_url: url for rabbitmq. it can be either '127.0.0.1' ("localhost") or 'amqp://*****:*****@10.12.7.22:5672/'
:param io_loop: io loop. if it is none, using IOLoop.current() instead.
"""
self._parameter = ConnectionParameters("127.0.0.1") if rabbitmq_url in ["localhost", "127.0.0.1"] else \
URLParameters(rabbitmq_url)
if io_loop is None:
io_loop = IOLoop.current()
self._io_loop = io_loop
self._logger = logging.getLogger(__name__)
self._publish_conn = _AsyncConnection(rabbitmq_url, io_loop)
self._receive_conn = _AsyncConnection(rabbitmq_url, io_loop)
self._rpc_exchange_dict = dict()
self._rpc_corr_id_dict = dict()
def reprocess():
# To use this script, fill in the credentials for rabbit. They should be found in app/settings.py
rabbit_url = 'amqp://<user>:<pass>@<host>:<port>/%2f'
connection = BlockingConnection(URLParameters(rabbit_url))
channel = connection.channel()
method, properties, body = channel.basic_get('Seft.Responses.Quarantine')
if method:
try:
print("Recovered quarantine message")
print("Headers:")
print(properties.headers)
# Uncomment if extra information is needed (payload is encrypted so it's not likely to be useful)
# print("Body:")
# print(body)
publisher = QueuePublisher(urls=[rabbit_url],
queue='Seft.Responses')
publisher.publish_message(body, headers=properties.headers)
print("Message successfully reprocessed")
channel.basic_ack(method.delivery_tag)
print("Message ACK")
except PublishMessageError as e:
print(e)
channel.basic_nack(method.delivery_tag)
else:
print('No message found on quarantine queue')
def __init__(self,
connection_string: str,
queue: str,
tasks=None,
loglevel="WARNING"):
self._queue = queue
self._tasks = tasks or {}
self._logger = logging.getLogger(queue)
stream_handler = logging.StreamHandler()
# TODO: сделать кастомнй форматер чтобы добавлять пробелы (https://stackoverflow.com/questions/6692248/python-logging-string-formatting/22707429)
stream_format = logging.Formatter(
'[NokiMQ] - %(asctime)s [%(name)s] %(levelname)s %(message)s'
)
stream_handler.setFormatter(stream_format)
self._logger.addHandler(stream_handler)
self._logger.setLevel(logging.getLevelName(loglevel))
self._logger.info(f'Connecting to "{connection_string}"')
connection = None
try:
connection = BlockingConnection(URLParameters(connection_string))
except:
self._logger.critical('Connection failed')
quit(-1)
self._logger.info('Connected!')
self._channel = connection.channel()
self._channel.queue_declare(queue)
def _connect(self):
try:
self.connection = BlockingConnection(
URLParameters(settings.EQ_RABBITMQ_URL))
except AMQPError as e:
logger.error("unable to open rabbit mq connection",
exc_info=e,
rabbit_url=settings.EQ_RABBITMQ_URL)
try:
self.connection = BlockingConnection(
URLParameters(settings.EQ_RABBITMQ_URL_SECONDARY))
except AMQPError as err:
logger.error("unable to open rabbit mq connection",
exc_info=e,
rabbit_url=settings.EQ_RABBITMQ_URL_SECONDARY)
raise err
def __init__(self,
amqp_url=None,
queue_name=None,
on_message_callback=None,
panic_event=None,
logger=None,
thread_name=None,
**kwargs):
self._connection = None
self._channel = None
self._consumer_tag = None
self._queue_name = queue_name
self._on_message_callback = on_message_callback
self._panic_event = panic_event
self._max_concurrent = kwargs.get("max_concurrent", 1)
self.logger = logger or logging.getLogger(__name__)
if "connection_info" in kwargs:
params = kwargs["connection_info"]
# Default to one attempt as the Plugin implements its own retry logic
params["connection_attempts"] = params.get("connection_attempts",
1)
self._connection_parameters = PikaClient(
**params).connection_parameters()
else:
self._connection_parameters = URLParameters(amqp_url)
super(RequestConsumer, self).__init__(name=thread_name)
def __init__(self, rabbitmq_url, io_loop, timeout=10):
self._parameters = ConnectionParameters("127.0.0.1") if rabbitmq_url in ["localhost", "127.0.0.1"] else \
URLParameters(rabbitmq_url)
self._io_loop = io_loop
self._timeout = timeout
self._connection_queue = Queue(maxsize=1)
self.current_status = self.INIT_STATUS
def __init__(self, rabbitmq_url, configuration: Config, io_loop=None):
super().__init__()
self._parameter = ConnectionParameters("127.0.0.1") \
if rabbitmq_url in ["localhost", "127.0.0.1"] else URLParameters(rabbitmq_url)
if io_loop is None:
io_loop = IOLoop.current()
self.io_loop = io_loop
self.configuration = configuration
self.pub_connection = AsyncConnection(rabbitmq_url, io_loop)
configpub = configuration["publish"].values()
self.pub_channels = self.create_channel('exchange',
self.pub_connection,
*configpub)
self.rcv_connection = AsyncConnection(rabbitmq_url, io_loop)
configrcv = configuration["receive"].values()
self.rcv_channels = self.create_channel('queue', self.rcv_connection,
*configrcv)
# print(self.rcv_channels)
self._rpc_corr_id_dict = dict()
def _run_amqp(self):
url = self.output_id.amqp_host_id.connection
connection = BlockingConnection(URLParameters(url))
channel = connection.channel()
result = channel.basic_publish(**self._generate_amqp_data())
_logger.debug(result)
connection.close()
def amqp() -> BlockingConnection:
try:
dsn: str = Config.amqp_url()
return BlockingConnection(parameters=URLParameters(dsn))
except AMQPConnectionError:
logging.error("Couldn't connect to the AMQP broker at: [%s]" % dsn)
sys.exit(2)
def __init__(self, ampq_url=None):
if not ampq_url:
ampq_url = self._build_ampq_url()
parameters = URLParameters(ampq_url)
self.channel = BlockingConnection(parameters).channel()
self.channel.confirm_delivery()
self.process_functions = {}
def connect(self):
""" Creating connection object """
url = self.get_url()
logger.info('Connecting to %s', url)
return SelectConnection(
URLParameters(url),
self.on_connection_open,
stop_ioloop_on_close=False
)
def __init__(self, url, listened_queue, request_exchange,
response_exchange, response_routing_key):
super(EchoServer, self).__init__()
self.connection_parameters = URLParameters(url)
self.listened_queue = listened_queue
self.request_exchange = request_exchange
self.response_exchange = response_exchange
self.response_routing_key = response_routing_key
self.channel = None
self.connection = None
self.event = Event()
def get_connection():
while True:
try:
connc = BlockingConnection(
URLParameters(
os.getenv("AMQP_URL",
"amqp://*****:*****@localhost:5672/")))
if connc.is_open:
break
except Exception:
pass
return connc
def test_new_pika_invalid_ca_cert(self):
"""Assert a ConfigurationException is raised when the CA can't be opened."""
params = URLParameters('amqps://myhost')
tls_conf = {
'keyfile': os.path.join(FIXTURES_DIR, 'key.pem'),
'certfile': os.path.join(FIXTURES_DIR, 'cert.pem'),
'ca_cert': os.path.join(FIXTURES_DIR, 'invalid_ca.pem'),
}
with mock.patch.dict(config.conf, {'tls': tls_conf}):
self.assertRaises(ConfigurationException, _session._configure_tls_parameters, params)
self.assertTrue(isinstance(params.ssl_options, _session.SSLOptions))
def __init__(self, url: str, exchange: str, routing_key: str):
try:
self.conn = BlockingConnection(parameters=URLParameters(url))
self.channel = self.conn.channel()
logging.info(f'Successfully connected to AMQP Broker at [{url}]')
self.exchnage = exchange
self.routing_key = routing_key
# Declare the queue
self.channel.queue_declare(routing_key, durable=True)
self.channel.confirm_delivery()
except AMQPConnectionError:
logging.error(f'Couldn\'t connect to the AMQP broker at [{url}]')
sys.exit(2)
def test_new_pika(self):
"""Assert configuring a cert and key results in a TLS connection with new pika versions."""
params = URLParameters("amqps://myhost")
tls_conf = {
"keyfile": os.path.join(FIXTURES_DIR, "key.pem"),
"certfile": os.path.join(FIXTURES_DIR, "cert.pem"),
"ca_cert": os.path.join(FIXTURES_DIR, "ca_bundle.pem"),
}
with mock.patch.dict(config.conf, {"tls": tls_conf}):
_session._configure_tls_parameters(params)
self.assertTrue(isinstance(params.ssl_options, _session.SSLOptions))
def _connect(self):
try:
logger.info('attempt to open connection',
server='primary',
category='rabbitmq')
return BlockingConnection(URLParameters(self.rabbitmq_url))
except AMQPError as e:
logger.error('unable to open connection',
exc_info=e,
server='primary',
category='rabbitmq')
try:
logger.info('attempt to open connection',
server='secondary',
category='rabbitmq')
return BlockingConnection(
URLParameters(self.rabbitmq_secondary_url))
except AMQPError as err:
logger.error('unable to open connection',
exc_info=e,
server='secondary',
category='rabbitmq')
raise err
def test_new_pika_invalid_key(self):
"""Assert a ConfigurationException is raised when the key can't be opened."""
params = URLParameters("amqps://myhost")
tls_conf = {
"keyfile": os.path.join(FIXTURES_DIR, "invalid_key.pem"),
"certfile": os.path.join(FIXTURES_DIR, "cert.pem"),
"ca_cert": os.path.join(FIXTURES_DIR, "ca_bundle.pem"),
}
with mock.patch.dict(config.conf, {"tls": tls_conf}):
self.assertRaises(ConfigurationException,
_session._configure_tls_parameters, params)
self.assertTrue(isinstance(params.ssl_options, _session.SSLOptions))
def connect(self):
"""This method connects to RabbitMQ, returning the connection handle.
When the connection is established, the on_connection_open method
will be invoked by pika.
:rtype: pika.SelectConnection
"""
url = self.get_url()
logger.info('Connecting to %s', url)
return SelectConnection(
parameters=URLParameters(url),
on_open_callback=self.on_connection_open,
on_open_error_callback=self.on_connection_open_error,
on_close_callback=self.on_connection_closed)
def __init__(self, data):
self.corr_id = str(uuid4())
self.json_input = data
self.response = None
url = environ.get('CLOUDAMQP_URL',
'amqps://*****:*****@woodpecker.rmq.cloudamqp.com/ahsmnsum')
params = URLParameters(url)
self.connection = BlockingConnection(params)
self.channel = self.connection.channel()
result = self.channel.queue_declare(queue='', exclusive=True)
self.callback_queue = result.method.queue
self.channel.basic_consume(
queue=self.callback_queue,
on_message_callback=self.on_response,
auto_ack=True)
def replica_channel(settings):
if current_transport is not RabbitMQTransport:
pytest.skip("Replica channel is implemented only for RabbitMQTransport.")
connection = BlockingConnection(
parameters=URLParameters(settings.CQRS['url']),
)
rabbit_mq_channel = connection.channel()
rabbit_mq_channel.queue_purge('replica')
rabbit_mq_channel.queue_purge('dead_letter_replica')
yield rabbit_mq_channel
connection.close()
def test_old_pika_approach_no_cert(self):
"""Assert if no cert is provided, no key is passed to pika either."""
params = URLParameters("amqps://")
tls_conf = {"keyfile": "key.pem", "certfile": None, "ca_cert": "ca_bundle.pem"}
expected_options = {
"keyfile": None,
"certfile": None,
"ca_certs": "ca_bundle.pem",
"cert_reqs": ssl.CERT_REQUIRED,
"ssl_version": ssl.PROTOCOL_TLSv1_2,
}
with mock.patch.dict(config.conf, {"tls": tls_conf}):
_session._configure_tls_parameters(params)
self.assertTrue(params.ssl)
self.assertEqual(params.ssl_options, expected_options)
def connect(self) -> None:
"""
For now we use a synchronous connection - caller is blocked until a
message is added to the queue. We might switch to asynch connections
should this incur noticable latencies.
"""
params = URLParameters(self.connection_settings)
connection = BlockingConnection(params)
channel = connection.channel()
# set up the Exchange (if it does not exist)
channel.exchange_declare(exchange=self.exchange,
exchange_type=self.exchange_type,
durable=True,
auto_delete=False)
self.connection = connection
self.channel = channel
def main(self):
url = environ.get(
'CLOUDAMQP_URL',
'amqps://*****:*****@woodpecker.rmq.cloudamqp.com/ahsmnsum'
)
params = URLParameters(url)
connection = BlockingConnection(params)
channel = connection.channel()
print('start')
query = Query()
query.connect()
while True:
inmessage = channel.basic_get("received", auto_ack=True)
sleep(0.2)
if inmessage[2] is not None:
print('ok')
while True:
self.data = query.select()
sleep(0.8)
def connect(cls, url):
f = Future()
def on_open(conn):
f.set_result(cls(conn))
def on_open_error(conn, err):
f.set_exception(AMQPConnectionError(err))
def on_close(conn):
LOGGER.debug('connection closed: %s', conn)
TornadoConnection(
URLParameters(url),
on_open_callback=on_open,
on_open_error_callback=on_open_error,
on_close_callback=on_close,
)
return f
