def _uninstrument_channel_functions(channel: Channel) -> None:
for function_name in _FUNCTIONS_TO_UNINSTRUMENT:
if not hasattr(channel, function_name):
continue
function = getattr(channel, function_name)
if hasattr(function, "_original_function"):
channel.__setattr__(function_name, function._original_function)
def _instrument_basic_publish(channel: Channel, tracer: Tracer) -> None:
original_function = getattr(channel, "basic_publish")
decorated_function = utils._decorate_basic_publish(
original_function, channel, tracer)
setattr(decorated_function, "_original_function", original_function)
channel.__setattr__("basic_publish", decorated_function)
channel.basic_publish = decorated_function
def on_message(channel: Channel, method: Basic.Deliver, props: BasicProperties,
body: bytes):
"""Called when a message is delivered from RabbitMQ."""
LOGGER.info(
f"Received message # {method.delivery_tag} from {props.app_id}")
LOGGER.info(f"Acknowledging message # {method.delivery_tag}")
channel.basic_ack(method.delivery_tag)
def __init__(self, connection, channel_number, transport=None):
# We need to do this before the channel is invoked and send_method is
# called
CallbackManager.instance().add(channel_number,
spec.Channel.OpenOk,
transport._on_rpc_complete)
Channel.__init__(self, connection, channel_number, None, transport)
self.basic_get_ = Channel.basic_get
def wrapped_consumer(
ch: Channel,
method: Basic.Deliver,
properties: pika.BasicProperties,
body: bytes,
) -> None:
assert method.delivery_tag is not None
callback([orjson.loads(body)])
ch.basic_ack(delivery_tag=method.delivery_tag)
def __init__(self, connection, channel_number, transport=None):
# We need to do this before the channel is invoked and send_method is
# called
connection.callbacks.add(channel_number, spec.Channel.OpenOk,
transport._on_rpc_complete)
Channel.__init__(self, connection, channel_number, None, transport)
self.basic_get_ = Channel.basic_get
self._consumers = {}
def ack_message(channel: Channel, delivery_tag: Any):
"""Note that `channel` must be the same pika channel instance via which
the message being ACKed was retrieved (AMQP protocol constraint).
"""
if channel.is_open:
channel.basic_ack(delivery_tag)
else:
# Channel is already closed, so we can't ACK this message;
# log and/or do something that makes sense for your app in this case.
pass
def __init__(self, connection, channel_number, transport=None):
# We need to do this before the channel is invoked and send_method is
# called
connection.callbacks.add(channel_number, spec.Channel.OpenOk, transport._on_rpc_complete)
connection.callbacks.add(channel_number, spec.Channel.CloseOk, transport._on_rpc_complete)
Channel.__init__(self, connection, channel_number, None, transport)
self.basic_get_ = Channel.basic_get
self._consumers = {}
self.open()
def _acknowledge_message(channel: Channel, delivery_tag: str):
"""
Acknowledge message
:param channel: Channel which message came from
:param delivery_tag: Message id
"""
logger.debug(f'Acknowledging message: {delivery_tag}')
if channel.is_open:
channel.basic_ack(delivery_tag)
def on_channel_open_callback(self, channel: Channel):
self.logger.info("Channel opened")
self.channel = channel
self.channel.add_on_close_callback(self.on_channel_closed_callback)
self.logger.info(f"Declaring exchange: {self.EXCHANGE}")
channel.exchange_declare(
exchange=self.EXCHANGE,
exchange_type=self.EXCHANGE_TYPE,
callback=self.on_exchange_declare_ok_callback,
)
def callback(ch: Channel, method, properties, body: bytes):
# TODO: Add logging message
xml_out: List = json.loads(body, encoding='utf-8')
self.buffer.extend(xml_out)
self.i += 1
if self.i >= self.max_buf_size:
df = pd.DataFrame(xml_out)
table = pa.Table.from_pandas(df)
pq.write_to_dataset(table, pq_root_path, filesystem=fs)
self.i = 0
self.buffer = []
ch.basic_ack(delivery_tag=method.delivery_tag)
def _callback(self, ch: Channel, method, props, body):
try:
corr_id = props.correlation_id
feature: Queue = None
self._lock.acquire()
feature = self._feature_map.get(corr_id, None)
self._lock.release()
if feature:
feature.put(body)
ch.basic_ack(method.delivery_tag)
except Exception as e:
pretty_logger.error("rpc client callback error {}".format(
traceback.format_exc()))
ch.basic_reject(method.delivery_tag, requeue=False)
def callback(ch: Channel, method, properties, body):
"""
body 是一个json文档，然后里面就一个链接的list，list里面就一个链接
:param ch:
:param method:
:param properties:
:param body:
:return:
"""
text = requests.get(json.loads(str(body, encoding="utf-8"))).text
self.mail_box.send(json.dumps([
text,
]))
ch.basic_ack(delivery_tag=method.delivery_tag)
def callback_files(ch: Channel, method, properties, body):
# print(f'received msg to parge {body}')
output = ''
with fs.open(body, 'rb') as xml_file:
# parse xml file
result = parser.parse(xml_file)
# for k in result.keys():
# result[k] = list(result[k])
# # serialize output dict
output = json.dumps(result)
# publish output
channel_parser.basic_publish(exchange=EXCHANGE, routing_key=QUEUE_PARSER, body=output)
ch.basic_ack(delivery_tag=method.delivery_tag)
def test_handling(self):
envelope = create_envelope()
handler = Handler(envelope)
self.assertEqual(handler.envelope, envelope)
handler.initialize(key='value', key2='value2')
handler.pre_handle()
self.assertRaises(NotImplementedError, handler.handle)
handler.post_handle()
envelope = create_envelope()
consumer = Consumer('localhost', 'test')
mock_connection = SelectConnection()
mock_channel = Channel(mock_connection, 10, None)
consumer.connection = mock_connection
consumer.channel = mock_channel
handler = MessageHandler(consumer, envelope)
self.assertEqual(handler.envelope, envelope)
self.assertEqual(handler.consumer, consumer)
self.assertEqual(handler.channel, mock_channel)
handler.initialize(key='value', key2='value2')
handler.pre_handle()
self.assertRaises(NotImplementedError, handler.handle)
handler.post_handle()
msg = 'some message'
with self.assertRaises(AbortHandling) as cm:
handler.abort(reason=msg)
self.assertEqual(cm.exception.reason, msg)
with self.assertRaises(SkipHandling) as cm:
handler.skip(reason=msg)
self.assertEqual(cm.exception.reason, msg)
with self.assertRaises(HandlingError) as cm:
handler.error(error_msg=msg)
self.assertEqual(cm.exception.error_msg, msg)
def _handle_ack(
self,
future: Future,
channel: Channel,
method_frame: pika.spec.Basic.Deliver,
_queue_name: str,
) -> None:
result = future.result()
if not isinstance(result, CallbackResult):
raise ValueError(
"Consumer's callback function must return a CallbackResult object."
)
if result.action == MQAction.ack:
channel.basic_ack(delivery_tag=method_frame.delivery_tag)
else:
channel.basic_nack(delivery_tag=method_frame.delivery_tag,
requeue=False)
def send_reply(
logger: Logger,
current_channel: channel.Channel,
basic_deliver: spec.Basic.Deliver,
properties: spec.BasicProperties,
xqueue_header: dict,
correct: bool = False,
score: int = 0,
msg: str = "",
xqueue_body: dict = None,
):
"""
Send a reply message and acknowledge received one.
:param logger: Logger object.
:param current_channel: Channel object.
:param basic_deliver: Object which has exchange, routing key,
delivery tag and a redelivered flag of the message.
:param properties: Message properties.
:param xqueue_header: Unique message header.
:param correct: xqueue_body parameter.
:param score: xqueue_body parameter.
:param msg: xqueue_body parameter.
:param xqueue_body: alternative way to pass values.
"""
if not xqueue_body:
xqueue_body = {
"correct": correct,
"score": score,
"msg": msg,
}
reply: dict = {"xqueue_header": xqueue_header, "xqueue_body": xqueue_body}
#logger.debug("Reply message: %s", reply)
print("Reply message", reply)
current_channel.basic_publish(
exchange="",
routing_key=properties.reply_to,
properties=BasicProperties(correlation_id=properties.correlation_id),
body=json.dumps(reply),
)
# Acknowledge message in queue
current_channel.basic_ack(delivery_tag=basic_deliver.delivery_tag)
def test_retry_policy(self):
envelope = create_envelope()
consumer = create_consumer()
mock_connection = SelectConnection()
mock_channel = Channel(mock_connection, 10, None)
consumer.connection = mock_connection
consumer.channel = mock_channel
retry_policy = RetryPolicy()
self.assertRaises(NotImplementedError,
retry_policy.retry,
envelope=envelope)
def connect_to_queue(self, channel: Channel, queue_name: str) -> Any:
"""Connect to the given queue on the given channel.
Args:
channel: The active channel to the RMQ server
queue_name: The queue to connect too
Returns:
A representation of the requested queue, holding data about the
number of messages, amongst other things
"""
return channel.queue_declare(queue=queue_name, passive=True)
def _rpc_response(self, ch: Channel, method, props, body):
resp = error(msg="invoke failed")
# invoke
try:
params: dict = json.loads(body)
func: str = params.get("func", None)
payload: dict = params.get("payload", None)
if not func:
resp = error(msg="request params must include func field")
elif func not in self._handle_func_map.keys():
resp = error(msg="no func match")
else:
pretty_logger.info("ready to invoke: {}".format(func))
result = self._handle_func_map[func](payload)
resp = success(result)
except Exception as e:
pretty_logger.error("{}".format(traceback.format_exc()))
resp = error(msg=traceback.format_exc())
# response
try:
ch.basic_publish(
exchange='',
routing_key=props.reply_to,
properties=pika.BasicProperties(correlation_id=props.correlation_id), # 验证字段
body=json.dumps(resp)
)
ch.basic_ack(delivery_tag=method.delivery_tag)
except Exception as e:
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=False)
pretty_logger.error("republish failed. err: {}".format(traceback.format_exc()))
def test_fixed_delay_limited_retries_policy(self):
consumer = create_consumer()
mock_connection = SelectConnection()
mock_channel = Channel(mock_connection, 10, None)
consumer.connection = mock_connection
consumer.channel = mock_channel
retry_policy = FixedDelayLimitedRetriesPolicy(consumer,
delay=10,
retries_limit=7,
retry_queue_suffix='s')
self.assertEqual(isinstance(retry_policy, LimitedRetriesPolicy), True)
self.assertEqual(retry_policy.retry_delays, tuple([10] * 7))
self.assertEqual(retry_policy.retry_queue_suffix, 's')
def test_fixed_delay_unlimited_retries_policy(self):
consumer = create_consumer()
mock_connection = SelectConnection()
mock_channel = Channel(mock_connection, 10, None)
consumer.connection = mock_connection
consumer.channel = mock_channel
retry_policy = FixedDelayUnlimitedRetriesPolicy(consumer,
10,
retry_queue_suffix='h')
self.assertEqual(isinstance(retry_policy, UnlimitedRetriesPolicy),
True)
self.assertEqual(retry_policy.initial_delay, 10)
self.assertEqual(retry_policy.max_delay, 10)
self.assertEqual(retry_policy.delay_incremented_by, 0)
self.assertEqual(retry_policy.retry_queue_suffix, 'h')
def test_publish(publisher, monkeypatch):
with monkeypatch.context() as m:
publisher._channel = Channel(SelectConnection(), 1, lambda: 1)
m.setattr(publisher._channel, '_state', Channel.OPEN)
mock_publish = Mock()
m.setattr(publisher._channel, 'basic_publish', mock_publish)
item = {'url': 'https://www.example.com/page'}
publisher._publish(item)
mock_publish.assert_called_once_with(
exchange='exchange',
routing_key='exchange.queue',
body=json.dumps(item, ensure_ascii=False),
properties=BasicProperties(app_id='app',
content_type='application/json',
delivery_mode=1),
mandatory=True)
def instrument_channel(
channel: Channel,
tracer_provider: Optional[TracerProvider] = None,
) -> None:
if not hasattr(channel, "_is_instrumented_by_opentelemetry"):
channel._is_instrumented_by_opentelemetry = False
if channel._is_instrumented_by_opentelemetry:
_LOG.warning(
"Attempting to instrument Pika channel while already instrumented!"
)
return
tracer = trace.get_tracer(__name__, __version__, tracer_provider)
if not hasattr(channel, "_impl"):
_LOG.error("Could not find implementation for provided channel!")
return
if channel._impl._consumers:
PikaInstrumentor._instrument_consumers(channel._impl._consumers,
tracer)
PikaInstrumentor._instrument_channel_functions(channel, tracer)
def ack(self, channel: Channel, method: MQ.Deliver) -> None:
channel.basic_ack(delivery_tag=method.delivery_tag)
def __init__(self, *args, **nargs):
Channel.__init__(self, *args, **nargs)
self.add_on_close_callback(self._my_on_close)
def queue_delete(self, *args, **nargs):
return Channel.queue_delete(self, *args, **nargs)
def sentence_preprocessor_consumer(ch: Channel, method, properties, body):
logger.debug("Recebido: " + body.decode(), extra={"received_args": body})
# Recupera as informações da tarefa
try:
task_info = json.loads(body.decode())
logger.debug("Recuperando tarefa: " + task_info["task"])
save_sentence_task: SaveSentenceTaskModel = SaveSentenceTaskModel.objects(
id=task_info["task"]).first()
if save_sentence_task is None:
raise Exception("Não foi encontrada nenhuma tarefa com o id " +
task_info["task"])
except Exception as e:
logger.error("Erro ao recuperar a tarefa",
exc_info=True,
extra={"received_args": body})
ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
return False
success_task = process_task(save_sentence_task)
datafile_import_task = save_sentence_task.parent
if datafile_import_task.status == "queued":
datafile_import_task.status = "in_progress"
datafile_import_task.save()
DataFileUploadTaskModel.objects(
id=datafile_import_task.id,
progress__lt=datafile_import_task.total).update_one(inc__progress=1)
if success_task:
ch.basic_ack(delivery_tag=method.delivery_tag)
else:
ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
datafile_import_task.reload("progress", "total")
if datafile_import_task.progress >= datafile_import_task.total:
tasks_with_fail = SaveSentenceTaskModel.objects(
parent=datafile_import_task, status="error").count()
logger.debug("Tarefa de importação concluida: " +
str(datafile_import_task.id))
if tasks_with_fail > 0:
datafile_import_task.status = "error"
datafile_import_task.error = "Algumas das sentenças não foram importadas com sucesso"
else:
datafile_import_task.status = "success"
datafile_import_task.save()
# Exclui as tarefas de processamento (Evitar dados duplicados)
SaveSentenceTaskModel.objects(parent=datafile_import_task,
error="").delete()
SaveSentenceTaskModel.objects(parent=datafile_import_task).update(
set__content="CONTEUDO REMOVIDO")
logger.debug("Tarefa concluida: " + task_info["task"])
return True
def _ch_on_open(self, ch: Channel) -> None:
self._ch = ch
ch.add_on_close_callback(self._ch_on_close)
ch.queue_declare(queue=self._queue_name,
callback=self._on_queue_declared)
class OnlyOneBasicGetTestCase(unittest.TestCase):
def setUp(self):
self.channel = Channel(MagicMock(Connection)(), 0, None)
self.channel._state = Channel.OPEN
self.callback = MagicMock()
def test_two_basic_get_with_callback(self):
self.channel.basic_get('test-queue', self.callback)
self.channel._on_getok(MagicMock(Method)(), MagicMock(Header)(), '')
self.channel.basic_get('test-queue', self.callback)
self.channel._on_getok(MagicMock(Method)(), MagicMock(Header)(), '')
self.assertEqual(self.callback.call_count, 2)
def test_two_basic_get_without_callback(self):
self.channel.basic_get('test-queue', self.callback)
with self.assertRaises(DuplicateGetOkCallback):
self.channel.basic_get('test-queue', self.callback)
def exchange_bind(self, *args, **nargs):
return Channel.exchange_bind(self, *args, **nargs)
def queue_declare(self, *args, **nargs):
return Channel.queue_declare(self, *args, **nargs)
def setUp(self):
self.channel = Channel(MagicMock(Connection)(), 0, None)
self.channel._state = Channel.OPEN
self.callback = MagicMock()
def setUp(self):
self.channel = Channel(MagicMock(Connection)(), 0, None)
self.channel._state = Channel.OPEN
self.callback = MagicMock()
def _open(self, frame):
Channel._open(self, frame)
self.transport.remove_reply(frame)
def _open(self, frame):
Channel._open(self, frame)
self.transport.remove_reply(frame)
def queue_bind(self, *args, **nargs):
return Channel.queue_bind(self, *args, **nargs)
class OnlyOneBasicGetTestCase(unittest.TestCase):
def setUp(self):
self.channel = Channel(MagicMock(Connection)(), 0, None)
self.channel._state = Channel.OPEN
self.callback = MagicMock()
def test_two_basic_get_with_callback(self):
self.channel.basic_get('test-queue', self.callback)
self.channel._on_getok(MagicMock(Method)(), MagicMock(Header)(), '')
self.channel.basic_get('test-queue', self.callback)
self.channel._on_getok(MagicMock(Method)(), MagicMock(Header)(), '')
self.assertEqual(self.callback.call_count, 2)
def test_two_basic_get_without_callback(self):
self.channel.basic_get('test-queue', self.callback)
with self.assertRaises(DuplicateGetOkCallback):
self.channel.basic_get('test-queue', self.callback)
def _on_remote_close(self, frame):
Channel._on_remote_close(self, frame)
raise AMQPChannelError(frame.method.reply_code,
frame.method.reply_text)
def reject(self, channel: Channel, method: MQ.Deliver) -> None:
channel.basic_reject(
delivery_tag=method.delivery_tag,
requeue=False)
def _on_remote_close(self, frame):
Channel._on_remote_close(self, frame)
raise AMQPChannelError(frame.method.reply_code,
frame.method.reply_text)
def exchange_delete(self, *args, **nargs):
return Channel.exchange_delete(self, *args, **nargs)
