class RabbitmqPublisherUsingAmqpStorm(AbstractPublisher):
# 使用amqpstorm包实现的mq操作。
# 实例属性没在init里面写，造成补全很麻烦，写在这里做类属性，方便pycharm补全
connection = amqpstorm.UriConnection
channel = amqpstorm.Channel
channel_wrapper_by_ampqstormbaic = AmqpStormBasic
queue = AmqpStormQueue
# noinspection PyAttributeOutsideInit
# @decorators.synchronized
def init_broker(self):
# username=app_config.RABBITMQ_USER, password=app_config.RABBITMQ_PASS, host=app_config.RABBITMQ_HOST, port=app_config.RABBITMQ_PORT, virtual_host=app_config.RABBITMQ_VIRTUAL_HOST, heartbeat=60 * 10
self.logger.warning(f'使用AmqpStorm包 链接mq')
self.connection = amqpstorm.UriConnection(
f'amqp://{frame_config.RABBITMQ_USER}:{frame_config.RABBITMQ_PASS}@{frame_config.RABBITMQ_HOST}:{frame_config.RABBITMQ_PORT}/{frame_config.RABBITMQ_VIRTUAL_HOST}?heartbeat={60 * 10}'
)
self.channel = self.connection.channel() # type:amqpstorm.Channel
self.channel_wrapper_by_ampqstormbaic = AmqpStormBasic(self.channel)
self.queue = AmqpStormQueue(self.channel)
self.queue.declare(queue=self._queue_name, durable=True)
# @decorators.tomorrow_threads(10)
@deco_mq_conn_error
def concrete_realization_of_publish(self, msg):
self.channel_wrapper_by_ampqstormbaic.publish(
exchange='',
routing_key=self._queue_name,
body=msg,
properties={'delivery_mode': 2},
)
# nb_print(msg)
@deco_mq_conn_error
def clear(self):
self.queue.purge(self._queue_name)
self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功')
@deco_mq_conn_error
def get_message_count(self):
# noinspection PyUnresolvedReferences
return self.queue.declare(queue=self._queue_name,
durable=True)['message_count']
# @deco_mq_conn_error
def close(self):
self.channel.close()
self.connection.close()
self.logger.warning('关闭rabbitpy包 链接mq')
def test_basic_get_invalid_parameter(self):
channel = Channel(0, FakeConnection(), 360)
channel.set_state(Channel.OPEN)
basic = Basic(channel)
self.assertRaisesRegexp(
exception.AMQPInvalidArgument,
'queue should be a string',
basic.get, None
)
self.assertRaisesRegexp(
exception.AMQPInvalidArgument,
'no_ack should be a boolean',
basic.get, '', 'travis-ci'
)
channel.consumer_tags.append('travis-ci')
self.assertRaisesRegexp(
exception.AMQPChannelError,
"Cannot call 'get' when channel "
"is set to consume",
basic.get, '', True, 'travis-ci'
)
def test_basic_publish_invalid_parameter(self):
channel = Channel(0, FakeConnection(), 360)
channel.set_state(Channel.OPEN)
basic = Basic(channel)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'body should be a string', basic.publish, None,
'')
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'routing_key should be a string',
basic.publish, '', None)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'exchange should be a string', basic.publish,
'', '', None)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'properties should be a dict or None',
basic.publish, '', '', '', [])
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'properties should be a dict or None',
basic.publish, '', '', '', 1)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'mandatory should be a boolean', basic.publish,
'', '', '', {}, None)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'immediate should be a boolean', basic.publish,
'', '', '', {}, True, None)
def test_basic_consume_invalid_parameter(self):
channel = Channel(0, FakeConnection(), 360)
channel.set_state(Channel.OPEN)
basic = Basic(channel)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'queue should be a string', basic.consume,
None, 1)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'consumer_tag should be a string',
basic.consume, None, '', 1)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'exclusive should be a boolean', basic.consume,
None, '', '', None)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'no_ack should be a boolean', basic.consume,
None, '', '', True, None)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'no_local should be a boolean', basic.consume,
None, '', '', True, True, None)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'arguments should be a dict or None',
basic.consume, None, '', '', True, True, True,
[])
def test_basic_cancel_invalid_parameter(self):
channel = Channel(0, FakeConnection(), 360)
channel.set_state(Channel.OPEN)
basic = Basic(channel)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'consumer_tag should be a string',
basic.cancel, None)
def test_basic_recover_invalid_parameter(self):
channel = Channel(0, FakeConnection(), 360)
channel.set_state(Channel.OPEN)
basic = Basic(channel)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'requeue should be a boolean', basic.recover,
None)
def __init__(self, channel_id, connection, rpc_timeout):
super(Channel, self).__init__(channel_id)
self.rpc = Rpc(self, timeout=rpc_timeout)
self._inbound = []
self._connection = connection
self.confirming_deliveries = False
self.consumer_callback = None
self.basic = Basic(self)
self.queue = Queue(self)
self.exchange = Exchange(self)
def init_broker(self):
# username=app_config.RABBITMQ_USER, password=app_config.RABBITMQ_PASS, host=app_config.RABBITMQ_HOST, port=app_config.RABBITMQ_PORT, virtual_host=app_config.RABBITMQ_VIRTUAL_HOST, heartbeat=60 * 10
self.logger.warning(f'使用AmqpStorm包 链接mq')
self.connection = amqpstorm.UriConnection(
f'amqp://{frame_config.RABBITMQ_USER}:{frame_config.RABBITMQ_PASS}@{frame_config.RABBITMQ_HOST}:{frame_config.RABBITMQ_PORT}/{frame_config.RABBITMQ_VIRTUAL_HOST}?heartbeat={60 * 10}'
)
self.channel = self.connection.channel() # type:amqpstorm.Channel
self.channel_wrapper_by_ampqstormbaic = AmqpStormBasic(self.channel)
self.queue = AmqpStormQueue(self.channel)
self.queue.declare(queue=self._queue_name, durable=True)
def test_basic_get_content_body_timeout_error(self):
body = ContentBody(value=self.message)
channel = Channel(0, FakeConnection(), 0.01)
channel.set_state(Channel.OPEN)
basic = Basic(channel)
uuid = channel.rpc.register_request([body.name])
self.assertRaisesRegexp(exception.AMQPChannelError,
'rpc requests .* \(.*\) took too long',
basic._get_content_body, uuid,
len(self.message))
def __init__(self, channel_id, connection, rpc_timeout):
super(Channel, self).__init__(channel_id)
self.consumer_callback = None
self.rpc = Rpc(self, timeout=rpc_timeout)
self._confirming_deliveries = False
self._connection = connection
self._inbound = []
self._basic = Basic(self)
self._exchange = Exchange(self)
self._tx = Tx(self)
self._queue = Queue(self)
def test_basic_publish_confirms_raises_on_timeout(self):
connection = FakeConnection()
channel = Channel(9, connection, 0.01)
channel._confirming_deliveries = True
channel.set_state(Channel.OPEN)
basic = Basic(channel)
self.assertRaisesRegexp(exception.AMQPChannelError,
'rpc requests .* \(.*\) took too long',
basic.publish,
body=self.message,
routing_key='travis-ci')
def __init__(self, channel_id, connection, rpc_timeout,
on_close_impl=None):
super(Channel, self).__init__(channel_id)
self.rpc = Rpc(self, timeout=rpc_timeout)
self._consumer_callbacks = {}
self._confirming_deliveries = False
self._connection = connection
self._on_close_impl = on_close_impl
self._inbound = []
self._basic = Basic(self, connection.max_frame_size)
self._exchange = Exchange(self)
self._tx = Tx(self)
self._queue = Queue(self)
def test_basic_publish_confirms_raises_on_invalid_frame(self):
def on_publish_return_invalid_frame(*_):
channel.rpc.on_frame(specification.Basic.Cancel())
connection = FakeConnection(on_write=on_publish_return_invalid_frame)
channel = Channel(9, connection, 0.01)
channel._confirming_deliveries = True
channel.set_state(Channel.OPEN)
basic = Basic(channel)
self.assertRaisesRegexp(exception.AMQPChannelError,
'rpc requests .* \(.*\) took too long',
basic.publish,
body=self.message,
routing_key='travis-ci')
def test_basic_reject_invalid_parameter(self):
channel = Channel(0, FakeConnection(), 360)
channel.set_state(Channel.OPEN)
basic = Basic(channel)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'delivery_tag should be an integer',
basic.reject, 'travis-ci')
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'delivery_tag should be an integer',
basic.reject, None)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'requeue should be a boolean', basic.reject, 1,
None)
def test_basic_qos_invalid_parameter(self):
channel = Channel(0, FakeConnection(), 360)
channel.set_state(Channel.OPEN)
basic = Basic(channel)
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'prefetch_count should be an integer',
basic.qos, 'travis-ci')
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'prefetch_size should be an integer',
basic.qos, 1, 'travis-ci')
self.assertRaisesRegexp(exception.AMQPInvalidArgument,
'global_ should be a boolean', basic.qos, 1, 1,
'travis-ci')
class Channel(BaseChannel):
"""RabbitMQ Channel Class."""
def __init__(self, channel_id, connection, rpc_timeout):
super(Channel, self).__init__(channel_id)
self.rpc = Rpc(self, timeout=rpc_timeout)
self._inbound = []
self._connection = connection
self.confirming_deliveries = False
self.consumer_callback = None
self.basic = Basic(self)
self.queue = Queue(self)
self.exchange = Exchange(self)
def __enter__(self):
return self
def __exit__(self, exception_type, exception_value, _):
if exception_value:
LOGGER.warning('Closing channel due to an unhandled exception: %s',
exception_type)
if not self.is_open:
return
self.close()
def __int__(self):
return self._channel_id
def open(self):
"""Open Channel.
:return:
"""
self._inbound = []
self._exceptions = []
self.set_state(self.OPENING)
self.rpc_request(pamqp_spec.Channel.Open())
self.set_state(self.OPEN)
def close(self, reply_code=0, reply_text=''):
"""Close Channel.
:param int reply_code:
:param str reply_text:
:return:
"""
LOGGER.debug('Channel #%d Closing.', self.channel_id)
if not compatibility.is_integer(reply_code):
raise AMQPInvalidArgument('reply_code should be an integer')
elif not compatibility.is_string(reply_text):
raise AMQPInvalidArgument('reply_text should be a string')
if not self._connection.is_open or not self.is_open:
self.remove_consumer_tag()
self.set_state(self.CLOSED)
return
self.set_state(self.CLOSING)
self.stop_consuming()
self.rpc_request(pamqp_spec.Channel.Close(
reply_code=reply_code,
reply_text=reply_text))
del self._inbound[:]
self.set_state(self.CLOSED)
LOGGER.debug('Channel #%d Closed.', self.channel_id)
def confirm_deliveries(self):
"""Set the channel to confirm that each message has been
successfully delivered.
:return:
"""
self.confirming_deliveries = True
confirm_frame = pamqp_spec.Confirm.Select()
return self.rpc_request(confirm_frame)
def on_frame(self, frame_in):
"""Handle frame sent to this specific channel.
:param pamqp.Frame frame_in: Amqp frame.
:return:
"""
if self.rpc.on_frame(frame_in):
return
if frame_in.name in CONTENT_FRAME:
self._inbound.append(frame_in)
elif frame_in.name == 'Basic.ConsumeOk':
self.add_consumer_tag(frame_in['consumer_tag'])
elif frame_in.name == 'Channel.Close':
self._close_channel(frame_in)
elif frame_in.name == 'Basic.Cancel':
LOGGER.warning('Received Basic.Cancel on consumer_tag: %s',
frame_in.consumer_tag.decode('utf-8'))
self.remove_consumer_tag(frame_in.consumer_tag)
elif frame_in.name == 'Basic.CancelOk':
self.remove_consumer_tag(frame_in.consumer_tag)
elif frame_in.name == 'Basic.Return':
self._basic_return(frame_in)
else:
LOGGER.error('Unhandled Frame: %s -- %s',
frame_in.name, dict(frame_in))
def start_consuming(self, to_tuple=True):
"""Start consuming events.
:param bool to_tuple: Should incoming messages be converted to
arguments before delivery.
:return:
"""
while self.consumer_tags and not self.is_closed:
self.process_data_events(to_tuple=to_tuple)
def stop_consuming(self):
"""Stop consuming events.
:return:
"""
if not self.consumer_tags:
return
for tag in self.consumer_tags:
self.basic.cancel(tag)
self.remove_consumer_tag()
def process_data_events(self, to_tuple=True):
"""Consume inbound messages.
This is only required when consuming messages. All other
events are automatically handled in the background.
:param bool to_tuple: Should incoming messages be converted to
arguments before delivery.
:return:
"""
if not self.consumer_callback:
raise AMQPChannelError('no consumer_callback defined')
for message in self.build_inbound_messages(break_on_empty=True):
if not to_tuple:
# noinspection PyCallingNonCallable
self.consumer_callback(message)
continue
# noinspection PyCallingNonCallable
self.consumer_callback(*message.to_tuple())
sleep(IDLE_WAIT)
def build_inbound_messages(self, break_on_empty=False, to_tuple=False):
"""Build messages in the inbound queue.
:param bool break_on_empty: Should we break the loop if there are
no more messages in the inbound queue.
:param bool to_tuple: Should incoming messages be converted to
arguments before delivery.
:rtype: :py:class:`generator`
"""
self.check_for_errors()
while not self.is_closed:
message = self._build_message()
if not message:
if break_on_empty:
break
self.check_for_errors()
sleep(IDLE_WAIT)
continue
if to_tuple:
yield message.to_tuple()
continue
yield message
def write_frame(self, frame_out):
"""Write a pamqp frame from the current channel.
:param pamqp_spec.Frame frame_out: A single pamqp frame.
:return:
"""
self.check_for_errors()
self._connection.write_frame(self.channel_id, frame_out)
def write_frames(self, multiple_frames):
"""Write multiple pamqp frames from the current channel.
:param list multiple_frames: A list of pamqp frames.
:return:
"""
self.check_for_errors()
self._connection.write_frames(self.channel_id,
multiple_frames)
def check_for_errors(self):
"""Check for errors.
:return:
"""
self._connection.check_for_errors()
if self._connection.is_closed:
self.set_state(self.CLOSED)
if not self.exceptions:
why = AMQPConnectionError('connection was closed')
self.exceptions.append(why)
if self.is_closed:
why = AMQPChannelError('channel was closed')
self.exceptions.append(why)
super(Channel, self).check_for_errors()
def rpc_request(self, frame_out):
"""Perform a RPC Request.
:param pamqp_spec.Frame frame_out: Amqp frame.
:rtype: dict
"""
with self.rpc.lock:
uuid = self.rpc.register_request(frame_out.valid_responses)
self.write_frame(frame_out)
return self.rpc.get_request(uuid)
def _close_channel(self, frame_in):
"""Close Channel.
:param pamqp_spec.Channel.Close frame_in: Amqp frame.
:return:
"""
self.remove_consumer_tag()
if frame_in.reply_code != 200:
message = 'Channel %d was closed by remote server: %s' % \
(self._channel_id, frame_in.reply_text.decode('utf-8'))
self._exceptions.append(AMQPChannelError(message))
del self._inbound[:]
self.set_state(self.CLOSED)
def _basic_return(self, frame_in):
"""Handle Basic Return and treat it as an error.
:param pamqp_spec.Return frame_in: Amqp frame.
:return:
"""
message = "Message not delivered: {0!s} ({1!s}) to queue" \
" '{2!s}' from exchange '{3!s}'" \
.format(frame_in.reply_text.decode('utf-8'),
frame_in.reply_code,
frame_in.routing_key,
frame_in.exchange)
self.exceptions.append(AMQPMessageError(message))
def _build_message(self):
"""Fetch and build a complete Message from the inbound queue.
:rtype: Message
"""
with self.lock:
if len(self._inbound) < 3:
return None
basic_deliver = self._inbound.pop(0)
if not isinstance(basic_deliver, pamqp_spec.Basic.Deliver):
LOGGER.warning('Received an out-of-order frame: %s was '
'expecting a Basic.Deliver frame.',
basic_deliver)
return None
content_header = self._inbound.pop(0)
if not isinstance(content_header, ContentHeader):
LOGGER.warning('Received an out-of-order frame: %s was '
'expecting a ContentHeader frame.',
content_header)
return None
body = self._build_message_body(content_header.body_size)
message = Message(channel=self,
body=body,
method=dict(basic_deliver),
properties=dict(content_header.properties))
return message
def _build_message_body(self, body_size):
"""Build the Message body from the inbound queue.
:rtype: str
"""
body = bytes()
while len(body) < body_size:
if not self._inbound:
sleep(IDLE_WAIT)
continue
body_piece = self._inbound.pop(0)
if not body_piece:
break
body += body_piece.value
return body