def _connect(self):
try:
logger.info("attempt to open connection",
server="primary",
category="rabbitmq")
return BlockingConnection(URLParameters(self.rabbitmq_url))
except AMQPError as e:
logger.error(
"unable to open connection",
exc_info=e,
server="primary",
category="rabbitmq",
)
try:
logger.info(
"attempt to open connection",
server="secondary",
category="rabbitmq",
)
return BlockingConnection(
URLParameters(self.rabbitmq_secondary_url))
except AMQPError as err:
logger.error(
"unable to open connection",
exc_info=e,
server="secondary",
category="rabbitmq",
)
raise err
def run(self):
self.connection = BlockingConnection(self.connection_parameters)
self.channel = self.connection.channel()
self.channel.queue_declare(queue=self.listened_queue,
exclusive=True,
durable=True,
passive=False,
auto_delete=True)
self.channel.queue_bind(queue=self.listened_queue,
exchange=self.request_exchange,
routing_key=self.listened_queue)
method_frame = None
while method_frame is None:
if self.event.is_set():
break
method_frame, header_frame, body = self.channel.basic_get(
self.listened_queue)
if method_frame:
self.channel.publish(exchange=self.response_exchange,
routing_key=self.response_routing_key,
body=body,
properties=header_frame)
self.channel.basic_ack(method_frame.delivery_tag)
method_frame = None
sleep(0.1)
def send_msg_example(queue: str, message: str):
"""Пример отправки сообщения в очередь
:param queue: название очереди
:param message: тело сообщения
"""
# подключаемся к серверу
connection = BlockingConnection(ConnectionParameters('localhost'))
channel = connection.channel()
# проверяем, что очередь сущетсвует, или создаем новую
# method = channel.queue_declare('') создаст временную очередь со случайным именем
channel.queue_declare(
queue=queue, # название
durable=True, # объявить устойчивой
)
# необязательно: создаём обменник и связываем с очередью
# channel.exchange_declare('logs', exchange_type='fanout')
# channel.queue_bind(method.method.queue, 'logs')
# с типом fanout при отправке сообщения routing_key можно не указывать
# отправляем сообщение
channel.basic_publish(
exchange='', # точка обмена
routing_key=queue, # имя очереди
body=message, # сообщение
properties=BasicProperties(
delivery_mode=2, # объявить устойчивым
)
)
connection.close()
def connect(self):
""" Connect to broker """
logger.debug("Connecting to AMQP broker {}:{}".format(self.host, self.port))
try:
context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
context.verify_mode = ssl.CERT_REQUIRED
context.load_verify_locations(cafile=self.caCertsFile)
context.load_cert_chain(self.certFile, keyfile=self.keyFile)
ssloptions = SSLOptions(context)
except Exception:
ssloptions = None
conn_params = ConnectionParameters(
host=self.host,
port=self.port,
ssl_options=ssloptions
)
if self.connection is not None:
logger.debug("Connect called on {}:{}, but connection already defined. Disconnecting and reconnecting".format(self.host, self.port))
self.disconnect()
self.connection = BlockingConnection(conn_params)
self.channel = self.connection.channel()
logger.debug("Connected to AMQP broker {}:{}".format(self.host, self.port))
def main():
connection = BlockingConnection(
parameters=URLParameters("amqp://*****:*****@localhost:5672/%2F")
)
props = BasicProperties(
content_type="application/json",
content_encoding="utf8",
delivery_mode=2,
)
ch = connection.channel()
ch.exchange_declare(exchange="worker.mm", exchange_type="topic")
ch.queue_declare(queue="splitter_action_request_q")
ch.queue_bind(
queue="splitter_action_request_q",
exchange="worker.mm",
routing_key="splitter.*.request",
)
ch.basic_publish(
exchange="worker.mm",
routing_key="splitter.split.request",
properties=props,
body=json.dumps(
{
"cache_dir": "/home/xy/misc/mtm/cache/youtube/好葉怎樣練習冥想完整教學 動畫講解-NLJcwbpkiJ0",
"action.type": "split",
"partial_duration": 2 * 60,
}
),
)
def create_connection():
connection = BlockingConnection(connection_parameters)
try:
yield connection
finally:
connection.close()
def main():
args = get_arguments()
credentials = None
if args.username and args.password:
credentials = PlainCredentials(args.username, args.password)
parameters = ConnectionParameters(host=args.host,
port=args.port,
credentials=credentials)
connection = BlockingConnection(parameters)
channel = connection.channel()
response = channel.queue_declare(exclusive=True, auto_delete=True)
queue = response.method.queue
channel.queue_bind(exchange=args.exchange,
queue=queue,
routing_key=args.routing_key)
channel.basic_consume(on_message, queue)
try:
channel.start_consuming()
except KeyboardInterrupt:
channel.stop_consuming()
connection.close()
def __init__(self):
self.conn = BlockingConnection(
ConnectionParameters(host=self.host, port=self.port))
self.channel = self.conn.channel()
self.channel.exchange_declare(exchange=self.exchange, type='topic')
def __init__(self, broker: Broker, rabbit_host: str):
self._logger = logging.getLogger(__name__)
self._rabbit_host = rabbit_host
self._broker = broker
self._broker.subscribe_broker(self)
self._broker_subscribers = []
# Init rabbit mq
self._logger.info(f"Connecting to rabbit host {rabbit_host}")
self._rabbit_connection = BlockingConnection(
ConnectionParameters(rabbit_host))
# self._rabbit_connection = pika.connection.Connection(pika.ConnectionParameters(rabbit_host))
self._rabbit_channel = self._rabbit_connection.channel()
for q in [
QueueName.TRADE_ACCOUNT, QueueName.ORDERS, QueueName.TRADES,
QueueName.MONEY_LIMITS, QueueName.STOCK_LIMITS,
QueueName.MSG_REPLY
]:
self._logger.info(f"Declaring rabbit queue {q}")
self._rabbit_channel.queue_declare(queue=q, durable=True)
# Subscribe to buy/sell events in new thread because pika consumes synchronously only
self._consumer_rabbit_connection = None
self._consumer_rabbit_channel = None
Thread(target=self.listen_commands).start()
self._logger.info("Initialized")
def main():
def on_message_receive_callback(channel, method, properties, body):
log("Mensaje recibido.")
message = json.loads(body)
operation = message["operation"]
image_processor.run(operation)
log("Todas las tareas asociadas a la operación '{}' han finalizado.".
format(operation))
log("Inicializando conexión a RabbitMQ en '{}:{}'...".format(
RABBITMQ_HOST, RABBITMQ_PORT))
connection = BlockingConnection(
ConnectionParameters(host=RABBITMQ_HOST,
port=RABBITMQ_PORT,
credentials=PlainCredentials(
RABBITMQ_USER, RABBITMQ_PASS),
virtual_host="/"))
channel = connection.channel()
channel.exchange_declare(exchange="XPyxel-Nodes",
exchange_type="direct",
durable=True)
channel.queue_declare(queue=WORKER_ID, durable=True)
channel.basic_consume(queue=WORKER_ID,
on_message_callback=on_message_receive_callback,
auto_ack=True)
log("Pyxel [{}] ha iniciado correctamente y está esperando peticiones. Presione [Ctrl]+C para salir..."
.format(WORKER_ID))
channel.start_consuming()
def _run_amqp(self):
url = self.output_id.amqp_host_id.connection
connection = BlockingConnection(URLParameters(url))
channel = connection.channel()
result = channel.basic_publish(**self._generate_amqp_data())
_logger.debug(result)
connection.close()
def open(self):
try:
self.connection = BlockingConnection(
ConnectionParameters(self.broker_url))
except:
sleep(1)
self.connection = BlockingConnection(
ConnectionParameters(self.broker_url))
self.channel = self.connection.channel()
self.channel.exchange_declare(exchange=broker_config.REQUEST_EXCHANGE,
exchange_type="topic")
self.channel.exchange_declare(exchange=broker_config.STATUS_EXCHANGE,
exchange_type="fanout")
def start_channel(self):
connection = BlockingConnection(ConnectionParameters(self.broker_host))
channel = connection.channel()
channel.basic_qos(prefetch_count=1)
channel.queue_declare(queue=self.queue, durable=True)
return channel
def __init__(self,
queue,
rabhost,
user,
passwd,
msg,
routing_key='py-fxp-publisher'):
self.queue = queue
self.host = rabhost
self.topic_type = 'direct'
self.user = user
self.passwd = passwd
self.result_exchange = 'py-fxp-publisher'
self.publish_connection = BlockingConnection(
ConnectionParameters(host=self.host,
port=5672,
virtual_host='/',
retry_delay=3,
connection_attempts=60,
credentials=PlainCredentials(
self.user, self.passwd)))
self.publish_channel = self.publish_connection.channel()
self.result_routing = routing_key
self.msg = msg
self.publish_channel.queue_declare(queue=self.queue,
passive=False,
durable=True,
exclusive=False,
auto_delete=False)
def connect_to_mq(server,
vhost,
user,
passwd,
ioloop=False,
on_open_callback=None):
partes = urlparse.urlparse(server)
result = None
if partes.scheme == '' or partes.scheme == 'rabbitmq': #Legado, usando RabbitMQ
server = partes.netloc or partes.path
cred = PlainCredentials(user, passwd)
if not ioloop:
result = BlockingConnection(
ConnectionParameters(str(server),
virtual_host=vhost,
credentials=cred))
else:
result = SelectConnection(
ConnectionParameters(str(server),
virtual_host=vhost,
credentials=cred), on_open_callback)
elif partes.scheme == 'nsq':
result = NsqConnection(partes.netloc,
vhost[1:] if len(vhost) > 1 else '')
return result
def __init__( self, host = 'localhost', port = 5672, user = '', password = '', vhost = '/', routingKey = ''):
credentials = PlainCredentials( user, password )
self._connection = BlockingConnection( ConnectionParameters( host, port, vhost, credentials ) )
#self._connection = SelectConnection( ConnectionParameters( host, port, vhost, credentials ) )
self._channel = self._connection.channel()
self._channel.exchange_declare( exchange = exchange_name, type = 'topic' )
self.rkey = routingKey
def send_msg_to_MQ(
msg_data): # Build connection -> build channel -> send message
#建立连接，然后发起通道，然后再发送信息
connection = BlockingConnection(
ConnectionParameters(host=HOST_NAME,
port=HOST_PORT,
virtual_host='/',
credentials=credentials))
channel = connection.channel()
result = channel.queue_declare(
queue='un_judged') # 声明消息队列，消息将在这个队列传递，如不存在，则创建
"""
data:
msg_data;
@key='TaskID' value->str # 自动生成唯一的任务ID (自动生成)
@key='studentNumber' value->str # 学号
@key='code' value->str # 需要评判的代码
@key='time' value->str # 当前的时间 (自动生成)
"""
TID = gen_task_ID(msg_data['studentNumber'])
message = json.dumps({
'TaskID': TID,
'code': msg_data['code'],
'time': current_datetime(),
'studentNumber': msg_data['studentNumber']
}) # build msg
channel.basic_publish(exchange='', routing_key='un_judged',
body=message) # 向队列插入数值 routing_key是队列名
connection.close()
return TID
def _get_consumer_rmq_objects(cls, host, port, creds, exchange, queue_name, prefetch_count):
connection = BlockingConnection(
ConnectionParameters(host=host, port=port, credentials=creds),
)
channel = connection.channel()
channel.basic_qos(prefetch_count=prefetch_count)
cls._declare_exchange(channel, exchange)
channel.queue_declare(queue_name, durable=True, exclusive=False)
for cqrs_id, replica_model in ReplicaRegistry.models.items():
channel.queue_bind(exchange=exchange, queue=queue_name, routing_key=cqrs_id)
# Every service must have specific SYNC routes
channel.queue_bind(
exchange=exchange,
queue=queue_name,
routing_key='cqrs.{}.{}'.format(queue_name, cqrs_id),
)
channel.basic_consume(
queue=queue_name,
on_message_callback=cls._consume_message,
auto_ack=False,
exclusive=False,
)
return connection, channel
def amq_channel(exchange):
conn = BlockingConnection(ConnectionParameters(get_broker_address()))
ch = conn.channel()
ch.confirm_delivery()
ch.exchange_declare(exchange=exchange, exchange_type='topic', durable=True)
yield ch
conn.close()
def is_alive(self):
try:
with BlockingConnection(
self.connection_parameters(connection_attempts=1)) as conn:
return conn.is_open
except AMQPError:
return False
def __init__(self, decider: BoboDecider, exchange_name: str, user_id: str,
parameters: ConnectionParameters,
max_sync_attempts: int) -> None:
super().__init__()
connection = BlockingConnection(parameters=parameters)
channel = connection.channel()
self.decider = decider
self.exchange_name = exchange_name
self.user_id = user_id
self.parameters = parameters
self.sync_id = None
self.max_sync_attempts = max_sync_attempts
self._queue_transition = Queue()
self._queue_clone = Queue()
self._queue_halt = Queue()
self._queue_final = Queue()
self._queue_action = Queue()
self._subs = []
self._connection = connection
self._channel = channel
self._is_synced = False
self._sync_response = None
def _connect(self):
try:
self.connection = BlockingConnection(
URLParameters(settings.EQ_RABBITMQ_URL))
except AMQPError as e:
logger.error("unable to open rabbit mq connection",
exc_info=e,
rabbit_url=settings.EQ_RABBITMQ_URL)
try:
self.connection = BlockingConnection(
URLParameters(settings.EQ_RABBITMQ_URL_SECONDARY))
except AMQPError as err:
logger.error("unable to open rabbit mq connection",
exc_info=e,
rabbit_url=settings.EQ_RABBITMQ_URL_SECONDARY)
raise err
def reprocess():
# To use this script, fill in the credentials for rabbit. They should be found in app/settings.py
rabbit_url = 'amqp://<user>:<pass>@<host>:<port>/%2f'
connection = BlockingConnection(URLParameters(rabbit_url))
channel = connection.channel()
method, properties, body = channel.basic_get('Seft.Responses.Quarantine')
if method:
try:
print("Recovered quarantine message")
print("Headers:")
print(properties.headers)
# Uncomment if extra information is needed (payload is encrypted so it's not likely to be useful)
# print("Body:")
# print(body)
publisher = QueuePublisher(urls=[rabbit_url],
queue='Seft.Responses')
publisher.publish_message(body, headers=properties.headers)
print("Message successfully reprocessed")
channel.basic_ack(method.delivery_tag)
print("Message ACK")
except PublishMessageError as e:
print(e)
channel.basic_nack(method.delivery_tag)
else:
print('No message found on quarantine queue')
def connect(self, consumer):
log.info("Connecting to RabbitMQ...")
user = self.settings["rabbit_user"]
password = self.settings["rabbit_password"]
queue = self.settings["backend_queue"]
parameters = dict(
host=self.settings["rabbit_host"],
port=self.settings["rabbit_port"],
socket_timeout=self.settings["rabbit_connecion_timeout"],
)
try:
credentials = PlainCredentials(user, password)
param = ConnectionParameters(**parameters)
self.connection = BlockingConnection(param)
self.channel = self.connection.channel()
self.channel.basic_consume(
queue=queue, auto_ack=True, on_message_callback=consumer
)
except Exception as e:
log.error("Something went wrong with connection to RabbitMQ... %s", e)
def _check_connection(cls):
if not getattr(thread_data, 'rabbitmq_connected', False) \
and 'oioioi.notifications' in settings.INSTALLED_APPS \
and NotificationHandler.last_connection_check < \
time.time() - NotificationHandler.conn_try_interval:
try:
o = six.moves.urllib.parse.urlparse(
settings.NOTIFICATIONS_RABBITMQ_URL)
kwargs = {}
if o.hostname:
kwargs['host'] = o.hostname
if o.port:
kwargs['port'] = o.port
if o.path:
kwargs['virtual_host'] = o.path
if o.username and o.password:
kwargs['credentials'] = PlainCredentials(o.username, o.password)
parameters = ConnectionParameters(**kwargs)
thread_data.conn = BlockingConnection(parameters)
thread_data.rabbitmq_connected = True
# pylint: disable=broad-except
except Exception:
NotificationHandler.last_connection_check = time.time()
logger.info("Notifications: Can't connect to RabbitMQ",
exc_info=True)
def __init__(
self,
host: str = Default.RABBITMQ_HOST,
port: int = Default.RABBITMQ_PORT,
task_queue: str = Default.TASK_QUEUE,
response_queue: str = Default.RESPONSE_QUEUE,
):
"""
Init rabbitmq publisher
:param host: rabbitmq host
:param port: rabbitmq port
:param task_queue: queue name
:param response_queue: response queue name
"""
self.task_queue_name = task_queue
self.response_queue_name = response_queue
self.connection = BlockingConnection(
ConnectionParameters(
host=host,
port=port,
))
self.channel = self.connection.channel()
self.task_queue = self.channel.queue_declare(
queue=self.task_queue_name)
self.response_queue = self.channel.queue_declare(
queue=self.response_queue_name, exclusive=True)
self.channel.basic_consume(
queue=self.response_queue_name,
on_message_callback=self.task_response,
auto_ack=True,
)
def amqp() -> BlockingConnection:
try:
dsn: str = Config.amqp_url()
return BlockingConnection(parameters=URLParameters(dsn))
except AMQPConnectionError:
logging.error("Couldn't connect to the AMQP broker at: [%s]" % dsn)
sys.exit(2)
def __init__(self,
connection_string: str,
queue: str,
tasks=None,
loglevel="WARNING"):
self._queue = queue
self._tasks = tasks or {}
self._logger = logging.getLogger(queue)
stream_handler = logging.StreamHandler()
# TODO: сделать кастомнй форматер чтобы добавлять пробелы (https://stackoverflow.com/questions/6692248/python-logging-string-formatting/22707429)
stream_format = logging.Formatter(
'[NokiMQ] - %(asctime)s [%(name)s] %(levelname)s %(message)s'
)
stream_handler.setFormatter(stream_format)
self._logger.addHandler(stream_handler)
self._logger.setLevel(logging.getLevelName(loglevel))
self._logger.info(f'Connecting to "{connection_string}"')
connection = None
try:
connection = BlockingConnection(URLParameters(connection_string))
except:
self._logger.critical('Connection failed')
quit(-1)
self._logger.info('Connected!')
self._channel = connection.channel()
self._channel.queue_declare(queue)
def send_msg_to_MQ(
msg_data): # Build connection -> build channel -> send message
#建立连接，然后发起通道，然后再发送信息
connection = BlockingConnection(
ConnectionParameters(host=HOST_NAME,
port=HOST_PORT,
virtual_host='/',
credentials=credentials))
channel = connection.channel()
result = channel.queue_declare(
queue='judged') # 声明消息队列，消息将在这个队列传递，如不存在，则创建
"""
data:
msg_data;
@key='TaskID' value->str # 任务ID
@key='studentNumber' value->str # 学号
@key='result' value->str # 代码评判和编译结果
@key='time' value->str # 代码提交的时间
"""
message = json.dumps({
'TaskID': msg_data['TaskID'],
'result': msg_data['result'],
'time': msg_data['time'],
'studentNumber': msg_data['studentNumber']
}) # build msg
channel.basic_publish(exchange='', routing_key='judged',
body=message) # 向队列插入数值 routing_key是队列名
connection.close()
def decrypt_and_write():
with open(settings.SDX_SEFT_CONSUMER_KEYS_FILE) as file:
keys = yaml.safe_load(file)
key_store = KeyStore(keys)
connection = BlockingConnection(URLParameters(settings.RABBIT_URL))
channel = connection.channel()
method, properties, body = channel.basic_get(
settings.RABBIT_QUARANTINE_QUEUE)
if method:
logger.info("Recovered quarantine message",
body=body,
headers=properties.headers)
try:
decrypted_message = decrypt(body.decode("utf-8"), key_store,
KEY_PURPOSE_CONSUMER)
payload = SeftConsumer.extract_file(decrypted_message,
properties.headers['tx_id'])
with open('/tmp/{}'.format(payload.file_name),
'wb') as recovered_file:
recovered_file.write(payload.decoded_contents)
channel.basic_ack(method.delivery_tag)
logger.info("Message ACK")
except (InvalidTokenException, ValueError):
logger.exception("Bad decrypt")
channel.basic_nack(method.delivery_tag)
logger.info("Nacking message")
except Exception:
logger.exception("Failed to process")
channel.basic_nack(method.delivery_tag)
logger.info("Nacking message")
else:
logger.info('No message found on quarantine queue')
