def __init__(self, url: str, exchange: str, routing_key: str):
try:
self.conn = BlockingConnection(parameters=URLParameters(url))
self.channel = self.conn.channel()
logging.info(f'Successfully connected to AMQP Broker at [{url}]')
self.exchnage = exchange
self.routing_key = routing_key
# Declare the queue
self.channel.queue_declare(routing_key, durable=True)
self.channel.confirm_delivery()
except AMQPConnectionError:
logging.error(f'Couldn\'t connect to the AMQP broker at [{url}]')
sys.exit(2)
def test_new_pika(self):
"""Assert configuring a cert and key results in a TLS connection with new pika versions."""
params = URLParameters('amqps://myhost')
tls_conf = {
'keyfile': os.path.join(FIXTURES_DIR, 'key.pem'),
'certfile': os.path.join(FIXTURES_DIR, 'cert.pem'),
'ca_cert': os.path.join(FIXTURES_DIR, 'ca_bundle.pem'),
}
with mock.patch.dict(config.conf, {'tls': tls_conf}):
_session._configure_tls_parameters(params)
self.assertTrue(isinstance(params.ssl_options, _session.SSLOptions))
def test_new_pika_invalid_ca_cert(self):
"""Assert a ConfigurationException is raised when the CA can't be opened."""
params = URLParameters("amqps://myhost")
tls_conf = {
"keyfile": os.path.join(FIXTURES_DIR, "key.pem"),
"certfile": os.path.join(FIXTURES_DIR, "cert.pem"),
"ca_cert": os.path.join(FIXTURES_DIR, "invalid_ca.pem"),
}
with mock.patch.dict(config.conf, {"tls": tls_conf}):
self.assertRaises(ConfigurationException,
_session._configure_tls_parameters, params)
self.assertTrue(isinstance(params.ssl_options, _session.SSLOptions))
def test_new_pika_invalid_key(self):
"""Assert a ConfigurationException is raised when the key can't be opened."""
params = URLParameters('amqps://myhost')
tls_conf = {
'keyfile': os.path.join(FIXTURES_DIR, 'invalid_key.pem'),
'certfile': os.path.join(FIXTURES_DIR, 'cert.pem'),
'ca_cert': os.path.join(FIXTURES_DIR, 'ca_bundle.pem'),
}
with mock.patch.dict(config.conf, {'tls': tls_conf}):
self.assertRaises(ConfigurationException,
_session._configure_tls_parameters, params)
self.assertTrue(isinstance(params.ssl_options, _session.SSLOptions))
def _connect(self):
try:
logger.info('attempt to open connection',
server='primary',
category='rabbitmq')
return BlockingConnection(URLParameters(self.rabbitmq_url))
except AMQPError as e:
logger.error('unable to open connection',
exc_info=e,
server='primary',
category='rabbitmq')
try:
logger.info('attempt to open connection',
server='secondary',
category='rabbitmq')
return BlockingConnection(
URLParameters(self.rabbitmq_secondary_url))
except AMQPError as err:
logger.error('unable to open connection',
exc_info=e,
server='secondary',
category='rabbitmq')
raise err
def connect(self):
"""This method connects to RabbitMQ, returning the connection handle.
When the connection is established, the on_connection_open method
will be invoked by pika.
:rtype: pika.SelectConnection
"""
url = self.get_url()
logger.info('Connecting to %s', url)
return SelectConnection(
parameters=URLParameters(url),
on_open_callback=self.on_connection_open,
on_open_error_callback=self.on_connection_open_error,
on_close_callback=self.on_connection_closed)
def _connect(self):
try:
logger.info("attempt to open connection",
server="primary",
category="rabbitmq")
self.connection = BlockingConnection(
URLParameters(self.rabbitmq_url))
except AMQPError as e:
logger.error("unable to open connection",
exc_info=e,
server="primary",
category="rabbitmq")
try:
logger.info("attempt to open connection",
server="secondary",
category="rabbitmq")
self.connection = BlockingConnection(
URLParameters(self.rabbitmq_secondary_url))
except AMQPError as err:
logger.error("unable to open connection",
exc_info=e,
server="secondary",
category="rabbitmq")
raise err
def replica_channel(settings):
if current_transport is not RabbitMQTransport:
pytest.skip("Replica channel is implemented only for RabbitMQTransport.")
connection = BlockingConnection(
parameters=URLParameters(settings.CQRS['url']),
)
rabbit_mq_channel = connection.channel()
rabbit_mq_channel.queue_purge('replica')
rabbit_mq_channel.queue_purge('dead_letter_replica')
yield rabbit_mq_channel
connection.close()
def __init__(self, data):
self.corr_id = str(uuid4())
self.json_input = data
self.response = None
url = environ.get('CLOUDAMQP_URL',
'amqps://*****:*****@woodpecker.rmq.cloudamqp.com/ahsmnsum')
params = URLParameters(url)
self.connection = BlockingConnection(params)
self.channel = self.connection.channel()
result = self.channel.queue_declare(queue='', exclusive=True)
self.callback_queue = result.method.queue
self.channel.basic_consume(
queue=self.callback_queue,
on_message_callback=self.on_response,
auto_ack=True)
def test_old_pika_approach_no_cert(self):
"""Assert if no cert is provided, no key is passed to pika either."""
params = URLParameters("amqps://")
tls_conf = {"keyfile": "key.pem", "certfile": None, "ca_cert": "ca_bundle.pem"}
expected_options = {
"keyfile": None,
"certfile": None,
"ca_certs": "ca_bundle.pem",
"cert_reqs": ssl.CERT_REQUIRED,
"ssl_version": ssl.PROTOCOL_TLSv1_2,
}
with mock.patch.dict(config.conf, {"tls": tls_conf}):
_session._configure_tls_parameters(params)
self.assertTrue(params.ssl)
self.assertEqual(params.ssl_options, expected_options)
def connect(self) -> None:
"""
For now we use a synchronous connection - caller is blocked until a
message is added to the queue. We might switch to asynch connections
should this incur noticable latencies.
"""
params = URLParameters(self.connection_settings)
connection = BlockingConnection(params)
channel = connection.channel()
# set up the Exchange (if it does not exist)
channel.exchange_declare(exchange=self.exchange,
exchange_type=self.exchange_type,
durable=True,
auto_delete=False)
self.connection = connection
self.channel = channel
def main(self):
url = environ.get(
'CLOUDAMQP_URL',
'amqps://*****:*****@woodpecker.rmq.cloudamqp.com/ahsmnsum'
)
params = URLParameters(url)
connection = BlockingConnection(params)
channel = connection.channel()
print('start')
query = Query()
query.connect()
while True:
inmessage = channel.basic_get("received", auto_ack=True)
sleep(0.2)
if inmessage[2] is not None:
print('ok')
while True:
self.data = query.select()
sleep(0.8)
def connect(cls, url):
f = Future()
def on_open(conn):
f.set_result(cls(conn))
def on_open_error(conn, err):
f.set_exception(AMQPConnectionError(err))
def on_close(conn):
LOGGER.debug('connection closed: %s', conn)
TornadoConnection(
URLParameters(url),
on_open_callback=on_open,
on_open_error_callback=on_open_error,
on_close_callback=on_close,
)
return f
def start(cls):
"""It starts all background workers"""
if cls.started:
return
cls.started = True
if len(cls.background_workers) < 1:
raise RuntimeError('Background Workers are not loaded!')
logger.info('Starting background workers...')
# worker connections
cls.workers_conn = []
# workers
cls.workers = []
for worker_class in cls.background_workers:
if issubclass(worker_class, ConsumerMixin):
# using kombu conn
conn = Connection(conf['RABBIT_MQ_URL'], heartbeat=4)
# creating worker and worker thread
worker = worker_class(conn, logger)
worker_thread = threading.Thread(target=worker.run)
elif issubclass(worker_class, threading.Thread):
# using standard blocking connection with pika
conn = BlockingConnection(
parameters=URLParameters(conf['RABBIT_MQ_URL']))
# creating worker and worker thread
worker = worker_class(conn, logger)
worker_thread = worker
else:
raise RuntimeError('Unsupported worker type!')
worker_thread.setName('BackgroundWorker %s' % worker_class)
worker_thread.start()
cls.workers_conn.append(conn)
cls.workers.append(worker)
logger.info('Worker %s started!' % worker_class)
logger.info('Background workers started!')
def __init__(self, rabbitmq_url, configuration, io_loop=None):
"""
An asynchronous RabbitMQ client, that use tornado to complete invoking.
It is an `all-in-one` RabbitMQ client, including following interfaces:
1. publish - publish message.
2. receive - consume messages from a queue. If received properties is not none, it publishes result back
to `reply_to` queue.
3. rpc - publish a message with replay_to properties, wait for answer message and return value.
Architecture:
This class encapsulate two async connections, one for publishing messages and one for consuming messages.
:param rabbitmq_url: url for RabbitMQ. It can be either '127.0.0.1' ("localhost") or
'amqp://*****:*****@10.12.7.22:5672/'
:param configuration: RabbitMQ configuration for both receiving and publishing.
It is separated by the keys `receive` and `publish`
:param io_loop: io loop. If it is none, using IOLoop.current() instead
"""
self._parameter = ConnectionParameters("127.0.0.1") \
if rabbitmq_url in ["localhost", "127.0.0.1"] else URLParameters(rabbitmq_url)
if io_loop is None:
io_loop = IOLoop.current()
self._io_loop = io_loop
self.configuration = configuration
self._publish_connection = AsyncConnection(rabbitmq_url, io_loop,
self.logger)
self._publish_channels = {
publish_configuration["exchange"]:
ChannelConfiguration(self._publish_connection, self.logger,
io_loop, **publish_configuration)
for publish_configuration in configuration["publish"].values()
}
self._receive_connection = AsyncConnection(rabbitmq_url, io_loop,
self.logger)
self._receive_channels = {
receive_configuration["queue"]:
ChannelConfiguration(self._receive_connection, self.logger,
io_loop, **receive_configuration)
for receive_configuration in configuration["receive"].values()
}
self._rpc_corr_id_dict = dict()
def test_old_pika_approach(self):
"""Assert if pika is pre-1.0.0, the TLS settings are applied."""
params = URLParameters("amqps://")
tls_conf = {
"keyfile": "key.pem",
"certfile": "cert.pem",
"ca_cert": "custom_ca_bundle.pem",
}
expected_options = {
"keyfile": "key.pem",
"certfile": "cert.pem",
"ca_certs": "custom_ca_bundle.pem",
"cert_reqs": ssl.CERT_REQUIRED,
"ssl_version": ssl.PROTOCOL_TLSv1_2,
}
with mock.patch.dict(config.conf, {"tls": tls_conf}):
_session._configure_tls_parameters(params)
self.assertTrue(params.ssl)
self.assertEqual(params.ssl_options, expected_options)
def test_old_pika_approach(self):
"""Assert if pika is pre-1.0.0, the TLS settings are applied."""
params = URLParameters('amqps://')
tls_conf = {
'keyfile': 'key.pem',
'certfile': 'cert.pem',
'ca_cert': 'custom_ca_bundle.pem',
}
expected_options = {
'keyfile': 'key.pem',
'certfile': 'cert.pem',
'ca_certs': 'custom_ca_bundle.pem',
'cert_reqs': ssl.CERT_REQUIRED,
'ssl_version': ssl.PROTOCOL_TLSv1_2,
}
with mock.patch.dict(config.conf, {'tls': tls_conf}):
_session._configure_tls_parameters(params)
self.assertTrue(params.ssl)
self.assertEqual(params.ssl_options, expected_options)
def test_old_pika_approach_no_cert(self):
"""Assert if no cert is provided, no key is passed to pika either."""
params = URLParameters('amqps://')
tls_conf = {
'keyfile': 'key.pem',
'certfile': None,
'ca_cert': 'ca_bundle.pem',
}
expected_options = {
'keyfile': None,
'certfile': None,
'ca_certs': 'ca_bundle.pem',
'cert_reqs': ssl.CERT_REQUIRED,
'ssl_version': ssl.PROTOCOL_TLSv1_2,
}
with mock.patch.dict(config.conf, {'tls': tls_conf}):
_session._configure_tls_parameters(params)
self.assertTrue(params.ssl)
self.assertEqual(params.ssl_options, expected_options)
def start(self):
"""Start worker."""
# Define connection
while True:
try:
self._connection = SelectConnection(
URLParameters(self._amqp_url),
on_open_callback=self.on_open_connection)
self._connection.ioloop.start()
# Catch a Keyboard Interrupt to make sure that the connection is closed cleanly
except KeyboardInterrupt:
# Gracefully close the connection
self._connection.close()
# Start the IOLoop again so Pika can communicate, it will stop on its own when the connection is closed
self._connection.ioloop.start()
except:
if (self._debug):
print(
" [!] RabbitMQ Host Unreachable. Reconnecting in {} seconds..."
.format(self._reconnection_time))
sleep(self._reconnection_time)
def main():
rabbitmq_user = os.environ.get("RABBITMQ_USER")
rabbitmq_pwd = os.environ.get("RABBITMQ_PWD")
rabbitmq_host = os.environ.get("RABBITMQ_HOST")
rabbitmq_port = os.environ.get("RABBITMQ_PORT")
rabbitmq_queue = os.environ.get("RABBITMQ_QUEUE")
url = "amqp://%s:%s@%s:%s" %(rabbitmq_user, rabbitmq_pwd, rabbitmq_host, rabbitmq_port)
params = URLParameters(url)
connection = BlockingConnection(params)
channel = connection.channel()
channel.queue_declare(queue=rabbitmq_queue)
def callback(ch, method, properties, body):
if isinstance(body, bytes):
body = body.decode("utf-8")
blur = FaceBlur(body, destinationPath(body))
blur.locateFaces()
blur.blurFaces()
blur.save()
channel.basic_consume(queue=rabbitmq_queue, on_message_callback=callback, auto_ack=True)
channel.start_consuming()
def process(self, files):
done = False
while not done:
try:
self._connection = BlockingConnection(URLParameters(self._amqp_url))
self._channel = self._connection.channel()
count = 0
for f in files:
file_already_processed = [ exists(f.replace('RAW', i)) for i in ['DESERT', 'OCEAN-SEA', 'FOREST-JUNGLE']]
# Only send imgs if they haven't been already processed.
if (not all(file_already_processed)):
data = self.img_to_base64(f)
message = self.prepare(data, f)
self.send_to_queue(message, self._task_queue)
count += 1
if (count == self._batch):
sleep(self._wait)
count = 0
self._connection.close()
done = True
except:
if (self._debug):
print(" [!] RabbitMQ Host Unreachable. Reconnecting in {} seconds...".format(self._reconnection_time))
sleep(self._reconnection_time)
def connect(self, ioloop=None, stop_ioloop=True):
logger.debug('Connecting to %s', self._url)
self.stop_ioloop = stop_ioloop
return TornadoConnection(URLParameters(self._url),
self.on_connection_open,
custom_ioloop=ioloop)
def create_connection(url):
parameters = URLParameters(url)
connection = BlockingConnection(parameters)
return connection
fun1 = parse.check_format_file()
fun2 = parse.insert_for_graph()
parse.disconnect()
if 'Query returned successfully' in fun1:
channel.queue_declare(queue='received')
channel.basic_publish(exchange='', routing_key='received', body=f'ok')
ch.basic_publish(
exchange='',
routing_key=props.reply_to,
properties=BasicProperties(correlation_id=props.correlation_id),
body=str(f'{fun1} | {fun2}'))
ch.basic_ack(delivery_tag=method.delivery_tag)
if __name__ == '__main__':
url = environ.get(
'CLOUDAMQP_URL',
'amqps://*****:*****@woodpecker.rmq.cloudamqp.com/ahsmnsum'
)
params = URLParameters(url)
connection = BlockingConnection(params)
channel = connection.channel()
channel.queue_declare(queue='send')
channel.basic_qos(prefetch_count=1)
channel.basic_consume(queue='send', on_message_callback=on_request)
print(" [x] Awaiting RPC requests")
channel.start_consuming()
def get_con(url):
con = TornadoConnection(URLParameters(url))
con.add_on_close_callback(reconnect)
return con
#!/usr/bin/env python
import pika
import sys
from pika import BlockingConnection, URLParameters
connection = BlockingConnection(
parameters=URLParameters("amqp://*****:*****@localhost:5672/%2F"))
channel = connection.channel()
channel.exchange_declare(exchange="topic_logs", exchange_type="topic")
routing_key = "anonymous.info"
message = " ".join(sys.argv[2:]) or "Hello World!"
channel.basic_publish(exchange="topic_logs",
routing_key=routing_key,
body=message)
print(" [x] Sent %r:%r" % (routing_key, message))
connection.close()
def connect(connection_url):
""" Create and return a fresh connection
"""
return BlockingConnection(URLParameters(connection_url))
class AutoInspect(object):
params = URLParameters(
'amqp://*****:*****@192.168.35.172:20555/%2F?heartbeat_interval=1')
properties = BasicProperties(content_type='application/json',
app_id='tom',
delivery_mode=1)
def auto_inspect(self):
neid = '100000000013414'
inspect_session_id = INSPECT_SESSION_ID
# inspect_msg = Message(channel, properties={
# 'content-type': 'application/json'
# })
# conn = Connection(
# userid='guest', password='******',
# virtual_host='/', host='192.168.35.172',
# port=20555)
items = [{
"executive": "kmScript",
"ctrlColId": "1000",
"function": "LINUX_MAINFRAME_linux_mainframe_cmd_check",
"configneid": "",
"param": ""
}, {
"executive": "kmScript",
"ctrlColId": "1005",
"function": "LINUX_LOGICALVOLUME_linux_logicalvolume_cmd_check",
"configneid": "",
"param": ""
}, {
"executive": "kmScript",
"ctrlColId": "1009",
"function": "LINUX_SWAPPARTITION_linux_swapparttion_cmd_check",
"configneid": "",
"param": ""
}, {
"executive": "kmScript",
"ctrlColId": "1010",
"function": "LINUX_VOLUMEGROUP_linux_volumegroup_cmd_check",
"configneid": "",
"param": ""
}]
body = {
"msgtype": "request",
"business": "kmCheckScript",
"body": {
"neid": "900000012103258",
"inspectSessionId": "{0}".format(inspect_session_id),
"items": items
},
"replyto": "itm.tom"
}
# body = {"msgtype":"request",
# "business":"cmdCtrl",
# "body":
# {
# "neid":"{0}".format(neid),
# "inspectSessionId":"{0}".format(inspect_session_id),
# "items": "[{0}]".format(items)
# }
# }
conn = BlockingConnection(self.params)
channel = conn.channel()
for i in xrange(10):
result = channel.basic_publish(exchange=AMQ_EXCHANGE,
routing_key=AMQ_ROUTINGKEY,
body=json.dumps(body),
properties=self.properties)
if result:
logging.info('delivery comfirmed')
logging.info('publish result: {0}'.format(result))
else:
logging.info('delivery not confirmed')
# time.sleep(40)
channel.close()
conn.close()
def connect(self):
url = os.environ.get('AMQP_URL', 'amqp://localhost')
parameters = URLParameters(url)
reactor.connectTCP(parameters.host, parameters.port,
ReconnectingPikaFactory(parameters, self))
def connect():
# credentials = PlainCredentials('guest', 'guest')
parameters = URLParameters(getenv('AMQP_URI'))
parameters.connection_attempts = 7
return BlockingConnection(parameters)