def send_message_to(
self,
exchange_name: str,
ch: BlockingChannel,
routing_key: str,
properties: BasicProperties,
body: bytes,
):
if properties.headers:
redelivery_count = properties.headers.get("redelivery_count", 0)
properties.headers["redelivery_count"] = redelivery_count + 1
else:
properties.headers = {"redelivery_count": 1}
self.printer.print_send_message_to(exchange_name, routing_key,
properties.headers)
ch.basic_publish(
exchange=exchange_name,
routing_key=routing_key,
body=body,
properties=properties,
)
return properties.headers
def _cbk_wl_monitor(self, workload: Workload, ch: BlockingChannel,
method: Basic.Deliver, _: BasicProperties,
body: bytes) -> None:
metric = json.loads(body.decode())
ch.basic_ack(method.delivery_tag)
if self._node_type == NodeType.IntegratedGPU:
item = BasicMetric(metric['llc_references'], metric['llc_misses'],
metric['instructions'], metric['cycles'],
metric['gpu_core_util'],
metric['gpu_core_freq'], metric['gpu_emc_util'],
metric['gpu_emc_freq'], workload.perf_interval)
if self._node_type == NodeType.CPU:
item = BasicMetric(metric['llc_references'], metric['llc_misses'],
metric['instructions'], metric['cycles'], 0, 0,
0, 0, workload.perf_interval)
logger = logging.getLogger(f'monitoring.metric.{workload}')
logger.debug(f'{metric} is given from ')
metric_que = workload.metrics
if len(metric_que) == self._metric_buf_size:
metric_que.pop()
metric_que.appendleft(item)
async def execute(
self,
channel: BlockingChannel,
method,
properties,
body: CallInfo,
):
task = self.tasks[body.func]
if not is_coroutine_callable(task):
raise Exception(f"{task.__name__} is not coroutine function.")
func_name = body.func
file = task.__code__.co_filename
line = task.__code__.co_firstlineno
try:
result = await task.do(**body.kwargs)
self.logger.info(
f"[SUCCESS]{file} {line} {func_name}(**{body.kwargs!r})")
except Exception as e:
self.logger.warning(
f"[FAILED]{file} {line} {func_name}(**{body.kwargs!r}) {e}")
# TODO: 処理が数回失敗した場合は、デッドレターキューに格納する（現在はとりあえず削除）
# channel.basic_reject(delivery_tag=method.delivery_tag, requeue=False) # メッセージを破棄する
# channel.basic_reject(delivery_tag=method.delivery_tag, requeue=True) # メッセージを再度キューイングする。実行順序は、多分最後になる
# channel.close() # コネクションやチャンネル開放時、未応答のキューは再度キューイングされる。順序はそのままらしいので、エラーが発生し続け、他の処理を妨害する恐れがある
if not self.auto_ack:
channel.basic_ack(delivery_tag=method.delivery_tag)
def callback(self, ch: BlockingChannel, method: spec.Basic.Deliver,
properties: spec.BasicProperties, body: bytes):
log.debug(' -> Received ch: %s meth: %s props: %s body: %s', ch,
method, properties, body)
b = body.decode()
try:
# General format: {req_id, action, host}
act = json.loads(b) # type: dict
log.debug(' -> Decoded to %s', act)
if 'action' not in act:
raise ValueError('"action" was not in decoded json object...')
if act['action'] not in self.ACTIONS:
raise ValueError('Action "{}" is not a valid action...'.format(
act['action']))
except (JSONDecodeError, TypeError, ValueError, AttributeError):
log.exception('Error decoding json for %s', b)
return ch.basic_nack(delivery_tag=method.delivery_tag,
requeue=False)
try:
run_act = self.ACTIONS[act['action']](**act)
if run_act:
log.debug('Acknowledging success to MQ')
return ch.basic_ack(delivery_tag=method.delivery_tag)
else:
log.debug('Acknowledging failure (try later) to MQ')
return ch.basic_nack(delivery_tag=method.delivery_tag)
except (InvalidHostException, AttributeError, ValueError) as e:
log.warning('Invalid host... Type: %s Msg: %s', type(e), str(e))
return ch.basic_nack(delivery_tag=method.delivery_tag,
requeue=False)
except Exception:
log.exception('Unknown exception while handling action call...')
return ch.basic_nack(delivery_tag=method.delivery_tag)
def task_process(self, channel: BlockingChannel, method: Basic.Deliver,
properties: BasicProperties, body: bytes) -> None:
"""
Process the received task
:param channel: channel
:param method: method
:param properties: task properties
:param body: task body
:return: None
"""
raw_body = loads(body.decode(encoding="utf-8"))
cases = raw_body.get("cases", {})
task = TaskItem(**raw_body.get("task", {}))
try:
results = list(self.manager.multi_case_runner(cases=cases))
for result in results:
TaskCrud.create_task_result(task, result or {})
task.set_success(
msg=f"Task done: {len(results)} out of {len(cases)} cases")
except Exception as cases_err:
task.set_error(msg=f"Task error: {str(cases_err)}")
TaskCrud.update_task(task)
logger.info(msg=f"Done task {task.task_id}")
channel.basic_publish(
exchange="",
routing_key=properties.reply_to,
properties=BasicProperties(
correlation_id=properties.correlation_id),
body=dumps(task.as_json()).encode(encoding="utf-8"),
)
channel.basic_ack(delivery_tag=method.delivery_tag)
def do_consume(channel: BlockingChannel) -> None:
events: List[Dict[str, Any]] = []
last_process = time.time()
max_processed: Optional[int] = None
self.is_consuming = True
# This iterator technique will iteratively collect up to
# batch_size events from the RabbitMQ queue (if present)
# before calling the callback with the batch. If not
# enough events are present, it will sleep for at most
# timeout seconds before calling the callback with the
# batch of events it has.
for method, properties, body in channel.consume(
queue_name, inactivity_timeout=timeout):
if body is not None:
assert method is not None
events.append(orjson.loads(body))
max_processed = method.delivery_tag
now = time.time()
if len(events) >= batch_size or (
timeout and now >= last_process + timeout):
if events:
assert max_processed is not None
try:
callback(events)
channel.basic_ack(max_processed, multiple=True)
except BaseException:
channel.basic_nack(max_processed, multiple=True)
raise
events = []
last_process = now
if not self.is_consuming:
break
def create_exchange_and_bind_queue(
channel: BlockingChannel,
exchange: str,
queue: str,
binding_key: str,
dead_letter=False,
):
queue_arguments = {}
if dead_letter:
# Note: If queue is already created with another RABBITMQ_MESSAGE_TTL, it will fail.
message_ttl = int(os.environ.get("RABBITMQ_MESSAGE_TTL",
"1000")) # 1 second
queue_arguments = {
"x-message-ttl": message_ttl,
"x-dead-letter-exchange": f"dlx-{exchange}",
}
try:
channel.exchange_declare(exchange=exchange,
exchange_type="topic",
durable=True)
result = channel.queue_declare(queue=queue,
arguments=queue_arguments,
durable=True)
channel.queue_bind(exchange=exchange,
queue=result.method.queue,
routing_key=binding_key)
except Exception as error:
raise TypeError(
f"RabbitMQEventPublisher: Cannot create the exchange ({exchange}) and bind to queue ({queue}) using given routing_key ({binding_key}) \n{error}"
)
def gracefully_stop_runner(channel: BlockingChannel, count: int):
runners = db.get_collection('corp-hq', 'runners')
all_runners = list(runners.find())
for i in range(count):
runner = all_runners[i]
channel.basic_publish(exchange='', routing_key=runner['name'], body='')
LOG.info('Request for runner "%s" to stop sent.', runner['name'])
def callback(ch: BlockingChannel, method, properties, body):
msg = str(body, "utf8").split(":")
print('# [%s] %s 메세지를 받았습니다. \n %r' %
(datetime.datetime.now(), msg[0], body))
time.sleep(int(str(msg[1])) / 10)
print(" # [%s] 완료했습니다." % datetime.datetime.now())
ch.basic_ack(delivery_tag=method.delivery_tag)
def do_publish(channel: BlockingChannel) -> None:
channel.basic_publish(
exchange='',
routing_key=queue_name,
properties=pika.BasicProperties(delivery_mode=2),
body=body)
statsd.incr(f"rabbitmq.publish.{queue_name}")
def _uninstrument_channel_functions(channel: BlockingChannel) -> None:
for function_name in _FUNCTIONS_TO_UNINSTRUMENT:
if not hasattr(channel, function_name):
continue
function = getattr(channel, function_name)
if hasattr(function, "_original_function"):
channel.__setattr__(function_name, function._original_function)
unwrap(channel, "basic_consume")
def wrapped_consumer(
ch: BlockingChannel,
method: Basic.Deliver,
properties: pika.BasicProperties,
body: bytes,
) -> None:
callback([orjson.loads(body)])
ch.basic_ack(delivery_tag=method.delivery_tag)
def send_message(channel: BlockingChannel, body: str):
"""
Send a basic message to the given channel
:param channel: The channel to send to.
:param body: The body to send.
"""
print(f"Sending message: {body}")
channel.basic_publish(exchange="", routing_key=QUEUE_NAME, body=body)
def on_rpc_response(ch: BlockingChannel, method, props, body):
if props.correlation_id == self.corr_id:
decoded_body = body.decode()
log_info('GET RPC REPLY',
queue=reply_queue,
correlation_id=self.corr_id,
body=decoded_body)
self.result = jsons.loads(decoded_body)
ch.basic_ack(delivery_tag=method.delivery_tag)
def _on_message(
channel: BlockingChannel,
method: Basic.Deliver,
properties: BasicProperties,
body: bytes,
) -> None:
quote: StockQuote = pickle.loads(body)
handler([quote])
channel.basic_ack(method.delivery_tag)
def send_message(channel: BlockingChannel, message: str) -> bool:
try:
channel.basic_publish(exchange='',
routing_key=settings.RABBITMQ_QUEUE,
body=bytes(message, encoding='utf-8'))
return True
except Exception as err:
print(err)
return False
def opened(channel: BlockingChannel) -> None:
while True:
(meta, _, message) = channel.basic_get(queue_name)
if message is None:
break
channel.basic_ack(meta.delivery_tag)
messages.append(message)
def _message_callback(self, ch: BlockingChannel, method: Basic.Deliver, _,
body: bytes):
try:
body_as_dict = RabbitMQBrokerConsumer._decode_body(body)
self._callback(body_as_dict)
except Exception as e:
print(f'Error during handling message {body}: {e}')
finally:
ch.basic_ack(delivery_tag=method.delivery_tag)
def on_message(self, ch: BlockingChannel, method, properties, body: str):
payload = json.loads(body)
response = self.callback(self.service, payload)
payload = {**payload, **response}
self.channel.basic_publish("",
self.queue_out,
body=json.dumps(payload),
mandatory=True)
ch.basic_ack(delivery_tag=method.delivery_tag)
def wrapped_consumer(ch: BlockingChannel, method: Basic.Deliver,
properties: pika.BasicProperties,
body: str) -> None:
try:
consumer(ch, method, properties, body)
ch.basic_ack(delivery_tag=method.delivery_tag)
except Exception as e:
ch.basic_nack(delivery_tag=method.delivery_tag)
raise e
def wrapped_consumer(ch: BlockingChannel,
method: Basic.Deliver,
properties: pika.BasicProperties,
body: str) -> None:
try:
consumer(ch, method, properties, body)
ch.basic_ack(delivery_tag=method.delivery_tag)
except Exception as e:
ch.basic_nack(delivery_tag=method.delivery_tag)
raise e
def on_request(self, channel: BlockingChannel, method, props, body):
request = dill.loads(body)
response = self.execute_request(request)
channel.basic_publish(exchange='',
routing_key=props.reply_to,
properties=pika.BasicProperties(correlation_id = \
props.correlation_id),
body=dill.dumps(response))
channel.basic_ack(delivery_tag=method.delivery_tag)
def req_callback(ch: BlockingChannel, method: spec.Basic.Deliver,
_: spec.BasicProperties, body: bytes):
body = load_object(body)
ack = callback(body)
if ack is None:
ack = True
if ack:
ch.basic_ack(delivery_tag=method.delivery_tag)
else:
ch.basic_nack(delivery_tag=method.delivery_tag)
def target_consuming_executor(task: TargetTask, channel: BlockingChannel) -> NoReturn:
"""
Starts endless consuming messages from RabbitMQ queue
:param task: Parameters for producer (routing key and queue label)
:param channel: Prepared blocking channel for consuming
:return:
"""
channel.basic_consume(queue=task.queue_label, on_message_callback=nxlog_callback)
channel.start_consuming()
logger.warning("Should appear only on exception. !!!")
def raw_answer_callback(ch: BlockingChannel,
method: spec.Basic.Deliver,
_: spec.BasicProperties, body: bytes):
body: AnsBody = load_object(body)
ack = answer_callback(body['id'], body['ans'])
if ack is None:
ack = True
if ack:
ch.basic_ack(delivery_tag=method.delivery_tag)
else:
ch.basic_nack(delivery_tag=method.delivery_tag)
def ack_message(self, channel: BlockingChannel, delivery_tag: int) -> None:
if channel.is_open:
logging.info(
"%s", f"Message (delivery_tag={delivery_tag}) acknowledged")
channel.basic_ack(delivery_tag)
else:
logging.info(
"%s",
(f"Message (delivery_tag={delivery_tag}) "
"NOT acknowledged (channel closed)"),
)
def nack_message(self, channel: BlockingChannel,
delivery_tag: int) -> None:
if channel.is_open:
logging.info("%s",
f"Message (delivery_tag={delivery_tag}) rejected")
channel.basic_nack(delivery_tag)
else:
logging.info(
"%s",
f"Message (delivery_tag={delivery_tag}) NOT rejected (channel closed)",
)
def cb(ch: BlockingChannel, method: pika.spec.Basic.Deliver,
properties: pika.spec.BasicProperties, body: bytes):
key = properties.correlation_id
try:
callback(ch, method, properties, body)
ch.basic_ack(delivery_tag=method.delivery_tag)
self.logger.info(f" [x] Message {key} processed!")
except Exception:
self.logger.info(
f" [-] failed to process message {key}: {traceback.format_exc()}"
)
def _instrument_basic_publish(
channel: BlockingChannel,
tracer: Tracer,
publish_hook: utils.HookT = utils.dummy_callback,
) -> None:
original_function = getattr(channel, "basic_publish")
decorated_function = utils._decorate_basic_publish(
original_function, channel, tracer, publish_hook)
setattr(decorated_function, "_original_function", original_function)
channel.__setattr__("basic_publish", decorated_function)
channel.basic_publish = decorated_function
def on_rpc_response(ch: BlockingChannel, method, props, body):
if props.correlation_id == self.corr_id:
decoded_body = body.decode()
self.result = jsons.loads(decoded_body)
print({
'action': 'get_rmq_rpc_reply',
'queue': reply_queue,
'correlation_id': self.corr_id,
'result': self.result
})
self.replied = True
ch.basic_ack(delivery_tag=method.delivery_tag)
def req_callback(ch: BlockingChannel, method: spec.Basic.Deliver,
_: spec.BasicProperties, body: bytes):
body: ReqBody = load_object(body)
ack = request_callback(
body['req'],
lambda answer: answer_callback(body['id'], answer))
if ack is None:
ack = True
if ack:
ch.basic_ack(delivery_tag=method.delivery_tag)
else:
ch.basic_nack(delivery_tag=method.delivery_tag)
