def connect_to_mq(server,
vhost,
user,
passwd,
ioloop=False,
on_open_callback=None):
partes = urlparse.urlparse(server)
result = None
if partes.scheme == '' or partes.scheme == 'rabbitmq': #Legado, usando RabbitMQ
server = partes.netloc or partes.path
cred = PlainCredentials(user, passwd)
if not ioloop:
result = BlockingConnection(
ConnectionParameters(str(server),
virtual_host=vhost,
credentials=cred))
else:
result = SelectConnection(
ConnectionParameters(str(server),
virtual_host=vhost,
credentials=cred), on_open_callback)
elif partes.scheme == 'nsq':
result = NsqConnection(partes.netloc,
vhost[1:] if len(vhost) > 1 else '')
return result
def test_should_unsubscribe_all_successfully(make_user_created_event):
given_any_user_id_1 = UserId("user_id_1")
given_any_user_id_2 = UserId("user_id_2")
given_any_topic = "topic"
def callback(ch, method, properties, body):
event = Event.from_json(body)
global received_events
received_events.append(event)
if isinstance(event, make_user_created_event().__class__):
assert event.user_id == "user_id_1"
ch.basic_ack(delivery_tag=method.delivery_tag)
event_manager = RabbitMQEventManager(
connection_parameters=ConnectionParameters(host="localhost"),
subscribers={given_any_topic: callback},
)
event_manager.send(given_any_topic,
make_user_created_event(user_id=given_any_user_id_1))
event_manager.unsubscribe_all()
sleep(0.8)
event_manager.send(given_any_topic,
make_user_created_event(user_id=given_any_user_id_2))
await_for_events()
assert len(received_events) >= 1
def send_msg_to_MQ(
msg_data): # Build connection -> build channel -> send message
#建立连接，然后发起通道，然后再发送信息
connection = BlockingConnection(
ConnectionParameters(host=HOST_NAME,
port=HOST_PORT,
virtual_host='/',
credentials=credentials))
channel = connection.channel()
result = channel.queue_declare(
queue='judged') # 声明消息队列，消息将在这个队列传递，如不存在，则创建
"""
data:
msg_data;
@key='TaskID' value->str # 任务ID
@key='studentNumber' value->str # 学号
@key='result' value->str # 代码评判和编译结果
@key='time' value->str # 代码提交的时间
"""
message = json.dumps({
'TaskID': msg_data['TaskID'],
'result': msg_data['result'],
'time': msg_data['time'],
'studentNumber': msg_data['studentNumber']
}) # build msg
channel.basic_publish(exchange='', routing_key='judged',
body=message) # 向队列插入数值 routing_key是队列名
connection.close()
def __init__(self, rabbitmq_url, configuration: Config, io_loop=None):
super().__init__()
self._parameter = ConnectionParameters("127.0.0.1") \
if rabbitmq_url in ["localhost", "127.0.0.1"] else URLParameters(rabbitmq_url)
if io_loop is None:
io_loop = IOLoop.current()
self.io_loop = io_loop
self.configuration = configuration
self.pub_connection = AsyncConnection(rabbitmq_url, io_loop)
configpub = configuration["publish"].values()
self.pub_channels = self.create_channel('exchange',
self.pub_connection,
*configpub)
self.rcv_connection = AsyncConnection(rabbitmq_url, io_loop)
configrcv = configuration["receive"].values()
self.rcv_channels = self.create_channel('queue', self.rcv_connection,
*configrcv)
# print(self.rcv_channels)
self._rpc_corr_id_dict = dict()
def connect(self):
""" Connect to broker """
logger.debug("Connecting to AMQP broker {}:{}".format(self.host, self.port))
try:
context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
context.verify_mode = ssl.CERT_REQUIRED
context.load_verify_locations(cafile=self.caCertsFile)
context.load_cert_chain(self.certFile, keyfile=self.keyFile)
ssloptions = SSLOptions(context)
except Exception:
ssloptions = None
conn_params = ConnectionParameters(
host=self.host,
port=self.port,
ssl_options=ssloptions
)
if self.connection is not None:
logger.debug("Connect called on {}:{}, but connection already defined. Disconnecting and reconnecting".format(self.host, self.port))
self.disconnect()
self.connection = BlockingConnection(conn_params)
self.channel = self.connection.channel()
logger.debug("Connected to AMQP broker {}:{}".format(self.host, self.port))
def main():
args = get_arguments()
credentials = None
if args.username and args.password:
credentials = PlainCredentials(args.username, args.password)
parameters = ConnectionParameters(host=args.host,
port=args.port,
credentials=credentials)
connection = BlockingConnection(parameters)
channel = connection.channel()
response = channel.queue_declare(exclusive=True, auto_delete=True)
queue = response.method.queue
channel.queue_bind(exchange=args.exchange,
queue=queue,
routing_key=args.routing_key)
channel.basic_consume(on_message, queue)
try:
channel.start_consuming()
except KeyboardInterrupt:
channel.stop_consuming()
connection.close()
def send_msg_to_MQ(
msg_data): # Build connection -> build channel -> send message
#建立连接，然后发起通道，然后再发送信息
connection = BlockingConnection(
ConnectionParameters(host=HOST_NAME,
port=HOST_PORT,
virtual_host='/',
credentials=credentials))
channel = connection.channel()
result = channel.queue_declare(
queue='un_judged') # 声明消息队列，消息将在这个队列传递，如不存在，则创建
"""
data:
msg_data;
@key='TaskID' value->str # 自动生成唯一的任务ID (自动生成)
@key='studentNumber' value->str # 学号
@key='code' value->str # 需要评判的代码
@key='time' value->str # 当前的时间 (自动生成)
"""
TID = gen_task_ID(msg_data['studentNumber'])
message = json.dumps({
'TaskID': TID,
'code': msg_data['code'],
'time': current_datetime(),
'studentNumber': msg_data['studentNumber']
}) # build msg
channel.basic_publish(exchange='', routing_key='un_judged',
body=message) # 向队列插入数值 routing_key是队列名
connection.close()
return TID
def __init__(
self,
host: str = Default.RABBITMQ_HOST,
port: int = Default.RABBITMQ_PORT,
task_queue: str = Default.TASK_QUEUE,
response_queue: str = Default.RESPONSE_QUEUE,
):
"""
Init rabbitmq publisher
:param host: rabbitmq host
:param port: rabbitmq port
:param task_queue: queue name
:param response_queue: response queue name
"""
self.task_queue_name = task_queue
self.response_queue_name = response_queue
self.connection = BlockingConnection(
ConnectionParameters(
host=host,
port=port,
))
self.channel = self.connection.channel()
self.task_queue = self.channel.queue_declare(
queue=self.task_queue_name)
self.response_queue = self.channel.queue_declare(
queue=self.response_queue_name, exclusive=True)
self.channel.basic_consume(
queue=self.response_queue_name,
on_message_callback=self.task_response,
auto_ack=True,
)
def __init__(self):
self.conn = BlockingConnection(
ConnectionParameters(host=self.host, port=self.port))
self.channel = self.conn.channel()
self.channel.exchange_declare(exchange=self.exchange, type='topic')
def main():
def on_message_receive_callback(channel, method, properties, body):
log("Mensaje recibido.")
message = json.loads(body)
operation = message["operation"]
image_processor.run(operation)
log("Todas las tareas asociadas a la operación '{}' han finalizado.".
format(operation))
log("Inicializando conexión a RabbitMQ en '{}:{}'...".format(
RABBITMQ_HOST, RABBITMQ_PORT))
connection = BlockingConnection(
ConnectionParameters(host=RABBITMQ_HOST,
port=RABBITMQ_PORT,
credentials=PlainCredentials(
RABBITMQ_USER, RABBITMQ_PASS),
virtual_host="/"))
channel = connection.channel()
channel.exchange_declare(exchange="XPyxel-Nodes",
exchange_type="direct",
durable=True)
channel.queue_declare(queue=WORKER_ID, durable=True)
channel.basic_consume(queue=WORKER_ID,
on_message_callback=on_message_receive_callback,
auto_ack=True)
log("Pyxel [{}] ha iniciado correctamente y está esperando peticiones. Presione [Ctrl]+C para salir..."
.format(WORKER_ID))
channel.start_consuming()
def open(self):
try:
self.connection = BlockingConnection(
ConnectionParameters(self.broker_url))
except:
sleep(1)
self.connection = BlockingConnection(
ConnectionParameters(self.broker_url))
self.channel = self.connection.channel()
self.channel.exchange_declare(exchange=broker_config.REQUEST_EXCHANGE,
exchange_type="topic")
self.channel.exchange_declare(exchange=broker_config.STATUS_EXCHANGE,
exchange_type="fanout")
def start_channel(self):
connection = BlockingConnection(ConnectionParameters(self.broker_host))
channel = connection.channel()
channel.basic_qos(prefetch_count=1)
channel.queue_declare(queue=self.queue, durable=True)
return channel
def __init__(self,
queue,
rabhost,
user,
passwd,
msg,
routing_key='py-fxp-publisher'):
self.queue = queue
self.host = rabhost
self.topic_type = 'direct'
self.user = user
self.passwd = passwd
self.result_exchange = 'py-fxp-publisher'
self.publish_connection = BlockingConnection(
ConnectionParameters(host=self.host,
port=5672,
virtual_host='/',
retry_delay=3,
connection_attempts=60,
credentials=PlainCredentials(
self.user, self.passwd)))
self.publish_channel = self.publish_connection.channel()
self.result_routing = routing_key
self.msg = msg
self.publish_channel.queue_declare(queue=self.queue,
passive=False,
durable=True,
exclusive=False,
auto_delete=False)
def connection_parameters(self, **kwargs):
"""Get ``ConnectionParameters`` associated with this client
Will construct a ``ConnectionParameters`` object using parameters
passed at initialization as defaults. Any parameters passed in
\*\*kwargs will override initialization parameters.
Args:
**kwargs: Overrides for specific parameters
Returns:
:obj:`pika.ConnectionParameters`: ConnectionParameters object
"""
credentials = PlainCredentials(
username=kwargs.get('user', self._user),
password=kwargs.get('password', self._password),
)
return ConnectionParameters(
host=kwargs.get('host', self._host),
port=kwargs.get('port', self._port),
ssl=kwargs.get('ssl_enabled', self._ssl_enabled),
ssl_options=kwargs.get('ssl_options', self._ssl_options),
virtual_host=kwargs.get('virtual_host', self._virtual_host),
connection_attempts=kwargs.get('connection_attempts',
self._connection_attempts),
heartbeat_interval=kwargs.get('heartbeat_interval',
self._heartbeat_interval),
credentials=credentials,
)
def test_parameters(self):
params = ConnectionParameters(socket_timeout=0.5,
retry_delay=0.1,
connection_attempts=3)
self.assertEqual(params.socket_timeout, 0.5)
self.assertEqual(params.retry_delay, 0.1)
self.assertEqual(params.connection_attempts, 3)
def connection_parameters(self, **kwargs):
"""Get ``ConnectionParameters`` associated with this client
Will construct a ``ConnectionParameters`` object using parameters
passed at initialization as defaults. Any parameters passed in
kwargs will override initialization parameters.
Args:
**kwargs: Overrides for specific parameters
Returns:
:obj:`pika.ConnectionParameters`: ConnectionParameters object
"""
credentials = PlainCredentials(
username=kwargs.get("user", self._user),
password=kwargs.get("password", self._password),
)
return ConnectionParameters(
host=kwargs.get("host", self._host),
port=kwargs.get("port", self._port),
ssl=kwargs.get("ssl_enabled", self._ssl_enabled),
ssl_options=kwargs.get("ssl_options", self._ssl_options),
virtual_host=kwargs.get("virtual_host", self._virtual_host),
connection_attempts=kwargs.get("connection_attempts",
self._connection_attempts),
heartbeat=kwargs.get(
"heartbeat", kwargs.get("heartbeat_interval",
self._heartbeat)),
blocked_connection_timeout=kwargs.get(
"blocked_connection_timeout",
self._blocked_connection_timeout),
credentials=credentials,
)
def connect(self, consumer):
log.info("Connecting to RabbitMQ...")
user = self.settings["rabbit_user"]
password = self.settings["rabbit_password"]
queue = self.settings["backend_queue"]
parameters = dict(
host=self.settings["rabbit_host"],
port=self.settings["rabbit_port"],
socket_timeout=self.settings["rabbit_connecion_timeout"],
)
try:
credentials = PlainCredentials(user, password)
param = ConnectionParameters(**parameters)
self.connection = BlockingConnection(param)
self.channel = self.connection.channel()
self.channel.basic_consume(
queue=queue, auto_ack=True, on_message_callback=consumer
)
except Exception as e:
log.error("Something went wrong with connection to RabbitMQ... %s", e)
def __init__(self, broker: Broker, rabbit_host: str):
self._logger = logging.getLogger(__name__)
self._rabbit_host = rabbit_host
self._broker = broker
self._broker.subscribe_broker(self)
self._broker_subscribers = []
# Init rabbit mq
self._logger.info(f"Connecting to rabbit host {rabbit_host}")
self._rabbit_connection = BlockingConnection(
ConnectionParameters(rabbit_host))
# self._rabbit_connection = pika.connection.Connection(pika.ConnectionParameters(rabbit_host))
self._rabbit_channel = self._rabbit_connection.channel()
for q in [
QueueName.TRADE_ACCOUNT, QueueName.ORDERS, QueueName.TRADES,
QueueName.MONEY_LIMITS, QueueName.STOCK_LIMITS,
QueueName.MSG_REPLY
]:
self._logger.info(f"Declaring rabbit queue {q}")
self._rabbit_channel.queue_declare(queue=q, durable=True)
# Subscribe to buy/sell events in new thread because pika consumes synchronously only
self._consumer_rabbit_connection = None
self._consumer_rabbit_channel = None
Thread(target=self.listen_commands).start()
self._logger.info("Initialized")
def connect(self):
if self.connected:
return
cred = PlainCredentials(USERNAME, PASSWORD)
params = ConnectionParameters(host=HOST, port=PORT, virtual_host=VHOST, credentials=cred)
self.connection = TornadoConnection(params, on_open_callback=self.on_connected)
self.connection.add_on_close_callback(callback=self.on_closed)
def __init__( self, host = 'localhost', port = 5672, user = '', password = '', vhost = '/', routingKey = ''):
credentials = PlainCredentials( user, password )
self._connection = BlockingConnection( ConnectionParameters( host, port, vhost, credentials ) )
#self._connection = SelectConnection( ConnectionParameters( host, port, vhost, credentials ) )
self._channel = self._connection.channel()
self._channel.exchange_declare( exchange = exchange_name, type = 'topic' )
self.rkey = routingKey
def amq_channel(exchange):
conn = BlockingConnection(ConnectionParameters(get_broker_address()))
ch = conn.channel()
ch.confirm_delivery()
ch.exchange_declare(exchange=exchange, exchange_type='topic', durable=True)
yield ch
conn.close()
def _get_consumer_rmq_objects(cls, host, port, creds, exchange, queue_name, prefetch_count):
connection = BlockingConnection(
ConnectionParameters(host=host, port=port, credentials=creds),
)
channel = connection.channel()
channel.basic_qos(prefetch_count=prefetch_count)
cls._declare_exchange(channel, exchange)
channel.queue_declare(queue_name, durable=True, exclusive=False)
for cqrs_id, replica_model in ReplicaRegistry.models.items():
channel.queue_bind(exchange=exchange, queue=queue_name, routing_key=cqrs_id)
# Every service must have specific SYNC routes
channel.queue_bind(
exchange=exchange,
queue=queue_name,
routing_key='cqrs.{}.{}'.format(queue_name, cqrs_id),
)
channel.basic_consume(
queue=queue_name,
on_message_callback=cls._consume_message,
auto_ack=False,
exclusive=False,
)
return connection, channel
def __init__(self, rabbitmq_url, io_loop=None):
"""
A Rabbitmq client, using tornado to complete asynchronous invoking.
It is an `everything-in-one` RabbitMQ client, including following interfaces:
- establish connection
- publish
- receive
- rpc
details:
1. establish connection
Every instance of `TornadoAdapter` has two rabbitmq connections. one for publish and the other for consumer.
2. publish
Publishing connection creates a channel when invoke publish method. After publishing message, it closes channel.
3. receive
It receives message from queue and process message. and publish message if necessary.
4. rpc
It invoke rpc call and wait result asynchronously. Of course, it supports timeout.
:param rabbitmq_url: url for rabbitmq. it can be either '127.0.0.1' ("localhost") or 'amqp://*****:*****@10.12.7.22:5672/'
:param io_loop: io loop. if it is none, using IOLoop.current() instead.
"""
self._parameter = ConnectionParameters("127.0.0.1") if rabbitmq_url in ["localhost", "127.0.0.1"] else \
URLParameters(rabbitmq_url)
if io_loop is None:
io_loop = IOLoop.current()
self._io_loop = io_loop
self._logger = logging.getLogger(__name__)
self._publish_conn = _AsyncConnection(rabbitmq_url, io_loop)
self._receive_conn = _AsyncConnection(rabbitmq_url, io_loop)
self._rpc_exchange_dict = dict()
self._rpc_corr_id_dict = dict()
def __init__(self, rabbitmq_url, io_loop, timeout=10):
self._parameters = ConnectionParameters("127.0.0.1") if rabbitmq_url in ["localhost", "127.0.0.1"] else \
URLParameters(rabbitmq_url)
self._io_loop = io_loop
self._timeout = timeout
self._connection_queue = Queue(maxsize=1)
self.current_status = self.INIT_STATUS
def _check_connection(cls):
if not getattr(thread_data, 'rabbitmq_connected', False) \
and 'oioioi.notifications' in settings.INSTALLED_APPS \
and NotificationHandler.last_connection_check < \
time.time() - NotificationHandler.conn_try_interval:
try:
o = six.moves.urllib.parse.urlparse(
settings.NOTIFICATIONS_RABBITMQ_URL)
kwargs = {}
if o.hostname:
kwargs['host'] = o.hostname
if o.port:
kwargs['port'] = o.port
if o.path:
kwargs['virtual_host'] = o.path
if o.username and o.password:
kwargs['credentials'] = PlainCredentials(o.username, o.password)
parameters = ConnectionParameters(**kwargs)
thread_data.conn = BlockingConnection(parameters)
thread_data.rabbitmq_connected = True
# pylint: disable=broad-except
except Exception:
NotificationHandler.last_connection_check = time.time()
logger.info("Notifications: Can't connect to RabbitMQ",
exc_info=True)
def connect(self):
"""This method connects to RabbitMQ, returning the connection handle.
When the connection is established, the on_connection_open method
will be invoked by pika.
:rtype: pika.SelectConnection
"""
LOGGER.info("Connecting to %s", self._server_details.host)
credentials = PlainCredentials(self._server_details.username,
self._server_details.password)
connection_params = ConnectionParameters(
host=self._server_details.host,
port=self._server_details.port,
virtual_host=self._server_details.vhost,
credentials=credentials,
)
if self._server_details.uses_ssl:
cafile = os.getenv("REQUESTS_CA_BUNDLE")
ssl_context = ssl.create_default_context(cafile=cafile)
connection_params.ssl_options = SSLOptions(ssl_context)
return SelectConnection(
parameters=connection_params,
on_open_callback=self.on_connection_open,
on_open_error_callback=self.on_connection_open_error,
on_close_callback=self.on_connection_closed,
)
def send_msg_example(queue: str, message: str):
"""Пример отправки сообщения в очередь
:param queue: название очереди
:param message: тело сообщения
"""
# подключаемся к серверу
connection = BlockingConnection(ConnectionParameters('localhost'))
channel = connection.channel()
# проверяем, что очередь сущетсвует, или создаем новую
# method = channel.queue_declare('') создаст временную очередь со случайным именем
channel.queue_declare(
queue=queue, # название
durable=True, # объявить устойчивой
)
# необязательно: создаём обменник и связываем с очередью
# channel.exchange_declare('logs', exchange_type='fanout')
# channel.queue_bind(method.method.queue, 'logs')
# с типом fanout при отправке сообщения routing_key можно не указывать
# отправляем сообщение
channel.basic_publish(
exchange='', # точка обмена
routing_key=queue, # имя очереди
body=message, # сообщение
properties=BasicProperties(
delivery_mode=2, # объявить устойчивым
)
)
connection.close()
def __init__(self,
host='localhost',
port=5672,
login='******',
password='******',
virtual_host='/',
loop=None,
**kwargs):
self.loop = loop if loop else ioloop.IOLoop.current()
self.future_store = common.FutureStore(loop=self.loop)
self.__credentials = PlainCredentials(
login, password) if login else ExternalCredentials()
self.__connection_parameters = ConnectionParameters(
host=host,
port=port,
credentials=self.__credentials,
virtual_host=virtual_host,
**kwargs)
self._channels = dict()
self._connection = None
self.__closing = None
self.__close_started = False
self.__write_lock = locks.Lock()
def __init__(self,
host: str = 'localhost',
port: int = 5672,
login: str = 'guest',
password: str = 'guest',
virtual_host: str = '/',
ssl: bool = False,
*,
loop=None,
**kwargs):
self.loop = loop if loop else asyncio.get_event_loop()
self.future_store = FutureStore(loop=self.loop)
self.__credentials = PlainCredentials(login,
password) if login else None
self.__connection_parameters = ConnectionParameters(
host=host,
port=port,
credentials=self.__credentials,
virtual_host=virtual_host,
ssl=ssl,
**kwargs)
self._channels = dict()
self._connection = None
self.__closing = None
self.__write_lock = asyncio.Lock(loop=self.loop)
def test_should_create_a_rabbitmq_event_manager_and_publish_two_different_events(
make_user_created_event, make_first_name_added_event, given_any_topic):
given_rabbitmq_local_connection_parameters = ConnectionParameters(
host="localhost")
global received_events
received_events = []
def callback(ch, method, properties, body):
event = Event.from_json(body)
global received_events
received_events.append(event)
if isinstance(event, make_user_created_event().__class__):
assert event.user_id == "user_id"
elif isinstance(event, make_first_name_added_event().__class__):
assert event.user_id == "user_id"
assert event.first_name == "Any User"
ch.basic_ack(delivery_tag=method.delivery_tag)
event_manager = RabbitMQEventManager(
connection_parameters=given_rabbitmq_local_connection_parameters,
subscribers={given_any_topic: callback},
)
event_manager.send(given_any_topic, make_user_created_event())
event_manager.send(given_any_topic, make_first_name_added_event())
await_for_events()
event_manager.unsubscribe_all()
assert len(received_events) >= 2
