def _shedual_task(self):
channel = RabbitMqFactory(
is_use_rabbitpy=0).get_rabbit_cleint().creat_a_channel()
channel.queue_declare(queue=self._queue_name, durable=True)
channel.basic_qos(prefetch_count=self._concurrent_num)
def callback(ch, method, properties, body):
body = body.decode()
self.logger.debug(
f'从rabbitmq的 [{self._queue_name}] 队列中 取出的消息是： {body}')
body = json.loads(body)
kw = {
'ch': ch,
'method': method,
'properties': properties,
'body': body
}
self._submit_task(kw)
channel.basic_consume(
callback,
queue=self._queue_name,
# no_ack=True
)
channel.start_consuming()
def init_broker(self):
self.logger.warning(f'使用pika 链接mq')
self.rabbit_client = RabbitMqFactory(
is_use_rabbitpy=0).get_rabbit_cleint()
self.channel = self.rabbit_client.creat_a_channel()
self.queue = self.channel.queue_declare(queue=self._queue_name,
durable=True)
def init_broker(self):
self.logger.warning(f'使用rabbitpy包 链接mq')
self.rabbit_client = RabbitMqFactory(
is_use_rabbitpy=1).get_rabbit_cleint()
self.channel = self.rabbit_client.creat_a_channel()
self.queue = self.channel.queue_declare(queue=self._queue_name,
durable=True)
self.logger.critical(
'rabbitpy 快速发布 有问题会丢失大量任务，如果使用 rabbitmq 建议设置中间件为 BrokerEnum.RABBITMQ_AMQPSTORM'
)
os._exit(444) # noqa
def _shedual_task(self):
# channel = RabbitMqFactory(is_use_rabbitpy=0).get_rabbit_cleint().creat_a_channel()
# channel.queue_declare(queue=self._queue_name, durable=True)
# channel.basic_qos(prefetch_count=self._concurrent_num)
def callback(ch, method, properties, body):
body = body.decode()
self.logger.debug(
f'从rabbitmq的 [{self._queue_name}] 队列中 取出的消息是： {body}')
body = json.loads(body)
kw = {
'ch': ch,
'method': method,
'properties': properties,
'body': body
}
self._submit_task(kw)
while True:
# 文档例子 https://github.com/pika/pika
try:
self.logger.warning(f'使用pika 链接mq')
self.rabbit_client = RabbitMqFactory(
is_use_rabbitpy=0).get_rabbit_cleint()
self.channel = self.rabbit_client.creat_a_channel()
self.rabbitmq_queue = self.channel.queue_declare(
queue=self._queue_name, durable=True)
self.channel.basic_consume(
callback,
queue=self._queue_name,
# no_ack=True
)
self.channel.start_consuming()
# Don't recover if connection was closed by broker
# except pikav0.exceptions.ConnectionClosedByBroker:
# break
# Don't recover on channel errors
except pikav0.exceptions.AMQPChannelError as e:
# break
self.logger.error(e)
continue
# Recover on all other connection errors
except pikav0.exceptions.AMQPConnectionError as e:
self.logger.error(e)
continue
class RabbitmqPublisherUsingRabbitpy(AbstractPublisher):
"""
使用rabbitpy包实现的。
"""
# noinspection PyAttributeOutsideInit
def init_broker(self):
self.logger.warning(f'使用rabbitpy包 链接mq')
self.rabbit_client = RabbitMqFactory(
is_use_rabbitpy=1).get_rabbit_cleint()
self.channel = self.rabbit_client.creat_a_channel()
self.queue = self.channel.queue_declare(queue=self._queue_name,
durable=True)
self.logger.critical(
'rabbitpy 快速发布 有问题会丢失大量任务，如果使用 rabbitmq 建议设置中间件为 BrokerEnum.RABBITMQ_AMQPSTORM'
)
os._exit(444) # noqa
# @decorators.tomorrow_threads(10)
@deco_mq_conn_error
def concrete_realization_of_publish(self, msg):
# noinspection PyTypeChecker
import time
# time.sleep(0.1)
print(
self.channel.basic_publish(
exchange='',
routing_key=self._queue_name,
body=msg,
properties={'delivery_mode': 2},
))
@deco_mq_conn_error
def clear(self):
self.channel.queue_purge(self._queue_name)
self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功')
@deco_mq_conn_error
def get_message_count(self):
# noinspection PyUnresolvedReferences
ch_raw_rabbity = self.channel.channel
return len(
rabbitpy.amqp_queue.Queue(ch_raw_rabbity,
self._queue_name,
durable=True))
# @deco_mq_conn_error
def close(self):
self.channel.close()
self.rabbit_client.connection.close()
self.logger.warning('关闭rabbitpy包 链接mq')
class RabbitmqPublisher(AbstractPublisher):
"""
使用pika实现的。
"""
# noinspection PyAttributeOutsideInit
def custom_init(self):
self._lock_for_pika = Lock()
# noinspection PyAttributeOutsideInit
def init_broker(self):
self.logger.warning(f'使用pika 链接mq')
self.rabbit_client = RabbitMqFactory(
is_use_rabbitpy=0).get_rabbit_cleint()
self.channel = self.rabbit_client.creat_a_channel()
self.queue = self.channel.queue_declare(queue=self._queue_name,
durable=True)
# noinspection PyAttributeOutsideInit
@deco_mq_conn_error
def concrete_realization_of_publish(self, msg):
with self._lock_for_pika: # 亲测pika多线程publish会出错
self.channel.basic_publish(
exchange='',
routing_key=self._queue_name,
body=msg,
properties=BasicProperties(
delivery_mode=2, # make message persistent 2(1是非持久化)
))
@deco_mq_conn_error
def clear(self):
self.channel.queue_purge(self._queue_name)
self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功')
@deco_mq_conn_error
def get_message_count(self):
with self._lock_for_pika:
queue = self.channel.queue_declare(queue=self._queue_name,
durable=True)
return queue.method.message_count
# @deco_mq_conn_error
def close(self):
self.channel.close()
self.rabbit_client.connection.close()
self.logger.warning('关闭pika包 链接')
def _shedual_task(self):
# noinspection PyTypeChecker
channel = RabbitMqFactory(is_use_rabbitpy=1).get_rabbit_cleint(
).creat_a_channel() # type: rabbitpy.AMQP #
channel.queue_declare(queue=self._queue_name, durable=True)
channel.basic_qos(prefetch_count=self._concurrent_num)
for message in channel.basic_consume(self._queue_name, no_ack=False):
body = message.body.decode()
# self.logger.debug(f'从rabbitmq {self._queue_name} 队列中 取出的消息是： {body}')
self._print_message_get_from_broker('rabbitmq', body)
kw = {
'message': message,
'body': json.loads(message.body.decode())
}
self._submit_task(kw)
class RabbitmqConsumer(AbstractConsumer):
"""
使用pika包实现的。
"""
BROKER_KIND = 4
# noinspection PyAttributeOutsideInit
def custom_init(self):
self._lock_for_pika = Lock()
def _shedual_task(self):
# channel = RabbitMqFactory(is_use_rabbitpy=0).get_rabbit_cleint().creat_a_channel()
# channel.queue_declare(queue=self._queue_name, durable=True)
# channel.basic_qos(prefetch_count=self._concurrent_num)
def callback(ch, method, properties, body):
body = body.decode()
self.logger.debug(
f'从rabbitmq的 [{self._queue_name}] 队列中 取出的消息是： {body}')
body = json.loads(body)
kw = {
'ch': ch,
'method': method,
'properties': properties,
'body': body
}
self._submit_task(kw)
while True:
# 文档例子 https://github.com/pika/pika
try:
self.logger.warning(f'使用pika 链接mq')
self.rabbit_client = RabbitMqFactory(
is_use_rabbitpy=0).get_rabbit_cleint()
self.channel = self.rabbit_client.creat_a_channel()
self.rabbitmq_queue = self.channel.queue_declare(
queue=self._queue_name, durable=True)
self.channel.basic_consume(
callback,
queue=self._queue_name,
# no_ack=True
)
self.channel.start_consuming()
# Don't recover if connection was closed by broker
# except pikav0.exceptions.ConnectionClosedByBroker:
# break
# Don't recover on channel errors
except pikav0.exceptions.AMQPChannelError as e:
# break
self.logger.error(e)
continue
# Recover on all other connection errors
except pikav0.exceptions.AMQPConnectionError as e:
self.logger.error(e)
continue
def _confirm_consume000(self, kw):
with self._lock_for_pika:
try:
kw['ch'].basic_ack(
delivery_tag=kw['method'].delivery_tag) # 确认消费
except AMQPError as e:
self.logger.error(f'pika确认消费失败 {e}')
def _confirm_consume(self, kw):
with self._lock_for_pika:
self.__ack_message_pika(kw['ch'], kw['method'].delivery_tag)
# kw['ch'].connection.add_callback_threadsafe(functools.partial(self.__ack_message_pika, kw['ch'], kw['method'].delivery_tag))
def _requeue(self, kw):
# kw['ch'].connection.add_callback_threadsafe(functools.partial(self.__nack_message_pika, kw['ch'], kw['method'].delivery_tag))
# with self._lock_for_pika:
# return kw['ch'].basic_nack(delivery_tag=kw['method'].delivery_tag) # 立即重新入队。
with self._lock_for_pika:
self.__nack_message_pika(kw['ch'], kw['method'].delivery_tag)
def __nack_message_pika(self, channelx, delivery_tagx):
"""Note that `channel` must be the same pika channel instance via which
the message being ACKed was retrieved (AMQP protocol constraint).
"""
if channelx.is_open:
channelx.basic_nack(delivery_tagx)
else:
# Channel is already closed, so we can't ACK this message;
# log and/or do something that makes sense for your app in this case.
self.logger.error(channelx.is_open)
pass
def __ack_message_pika(self, channelx, delivery_tagx):
"""Note that `channel` must be the same pika channel instance via which
the message being ACKed was retrieved (AMQP protocol constraint).
"""
if channelx.is_open:
channelx.basic_ack(delivery_tagx)
else:
# Channel is already closed, so we can't ACK this message;
# log and/or do something that makes sense for your app in this case.
self.logger.error(channelx.is_open)
pass
