def send_message_to( self, exchange_name: str, ch: BlockingChannel, routing_key: str, properties: BasicProperties, body: bytes, ): if properties.headers: redelivery_count = properties.headers.get("redelivery_count", 0) properties.headers["redelivery_count"] = redelivery_count + 1 else: properties.headers = {"redelivery_count": 1} self.printer.print_send_message_to(exchange_name, routing_key, properties.headers) ch.basic_publish( exchange=exchange_name, routing_key=routing_key, body=body, properties=properties, ) return properties.headers
def _cbk_wl_monitor(self, workload: Workload, ch: BlockingChannel, method: Basic.Deliver, _: BasicProperties, body: bytes) -> None: metric = json.loads(body.decode()) ch.basic_ack(method.delivery_tag) if self._node_type == NodeType.IntegratedGPU: item = BasicMetric(metric['llc_references'], metric['llc_misses'], metric['instructions'], metric['cycles'], metric['gpu_core_util'], metric['gpu_core_freq'], metric['gpu_emc_util'], metric['gpu_emc_freq'], workload.perf_interval) if self._node_type == NodeType.CPU: item = BasicMetric(metric['llc_references'], metric['llc_misses'], metric['instructions'], metric['cycles'], 0, 0, 0, 0, workload.perf_interval) logger = logging.getLogger(f'monitoring.metric.{workload}') logger.debug(f'{metric} is given from ') metric_que = workload.metrics if len(metric_que) == self._metric_buf_size: metric_que.pop() metric_que.appendleft(item)
async def execute( self, channel: BlockingChannel, method, properties, body: CallInfo, ): task = self.tasks[body.func] if not is_coroutine_callable(task): raise Exception(f"{task.__name__} is not coroutine function.") func_name = body.func file = task.__code__.co_filename line = task.__code__.co_firstlineno try: result = await task.do(**body.kwargs) self.logger.info( f"[SUCCESS]{file} {line} {func_name}(**{body.kwargs!r})") except Exception as e: self.logger.warning( f"[FAILED]{file} {line} {func_name}(**{body.kwargs!r}) {e}") # TODO: 処理が数回失敗した場合は、デッドレターキューに格納する(現在はとりあえず削除) # channel.basic_reject(delivery_tag=method.delivery_tag, requeue=False) # メッセージを破棄する # channel.basic_reject(delivery_tag=method.delivery_tag, requeue=True) # メッセージを再度キューイングする。実行順序は、多分最後になる # channel.close() # コネクションやチャンネル開放時、未応答のキューは再度キューイングされる。順序はそのままらしいので、エラーが発生し続け、他の処理を妨害する恐れがある if not self.auto_ack: channel.basic_ack(delivery_tag=method.delivery_tag)
def callback(self, ch: BlockingChannel, method: spec.Basic.Deliver, properties: spec.BasicProperties, body: bytes): log.debug(' -> Received ch: %s meth: %s props: %s body: %s', ch, method, properties, body) b = body.decode() try: # General format: {req_id, action, host} act = json.loads(b) # type: dict log.debug(' -> Decoded to %s', act) if 'action' not in act: raise ValueError('"action" was not in decoded json object...') if act['action'] not in self.ACTIONS: raise ValueError('Action "{}" is not a valid action...'.format( act['action'])) except (JSONDecodeError, TypeError, ValueError, AttributeError): log.exception('Error decoding json for %s', b) return ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) try: run_act = self.ACTIONS[act['action']](**act) if run_act: log.debug('Acknowledging success to MQ') return ch.basic_ack(delivery_tag=method.delivery_tag) else: log.debug('Acknowledging failure (try later) to MQ') return ch.basic_nack(delivery_tag=method.delivery_tag) except (InvalidHostException, AttributeError, ValueError) as e: log.warning('Invalid host... Type: %s Msg: %s', type(e), str(e)) return ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) except Exception: log.exception('Unknown exception while handling action call...') return ch.basic_nack(delivery_tag=method.delivery_tag)
def task_process(self, channel: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes) -> None: """ Process the received task :param channel: channel :param method: method :param properties: task properties :param body: task body :return: None """ raw_body = loads(body.decode(encoding="utf-8")) cases = raw_body.get("cases", {}) task = TaskItem(**raw_body.get("task", {})) try: results = list(self.manager.multi_case_runner(cases=cases)) for result in results: TaskCrud.create_task_result(task, result or {}) task.set_success( msg=f"Task done: {len(results)} out of {len(cases)} cases") except Exception as cases_err: task.set_error(msg=f"Task error: {str(cases_err)}") TaskCrud.update_task(task) logger.info(msg=f"Done task {task.task_id}") channel.basic_publish( exchange="", routing_key=properties.reply_to, properties=BasicProperties( correlation_id=properties.correlation_id), body=dumps(task.as_json()).encode(encoding="utf-8"), ) channel.basic_ack(delivery_tag=method.delivery_tag)
def do_consume(channel: BlockingChannel) -> None: events: List[Dict[str, Any]] = [] last_process = time.time() max_processed: Optional[int] = None self.is_consuming = True # This iterator technique will iteratively collect up to # batch_size events from the RabbitMQ queue (if present) # before calling the callback with the batch. If not # enough events are present, it will sleep for at most # timeout seconds before calling the callback with the # batch of events it has. for method, properties, body in channel.consume( queue_name, inactivity_timeout=timeout): if body is not None: assert method is not None events.append(orjson.loads(body)) max_processed = method.delivery_tag now = time.time() if len(events) >= batch_size or ( timeout and now >= last_process + timeout): if events: assert max_processed is not None try: callback(events) channel.basic_ack(max_processed, multiple=True) except BaseException: channel.basic_nack(max_processed, multiple=True) raise events = [] last_process = now if not self.is_consuming: break
def create_exchange_and_bind_queue( channel: BlockingChannel, exchange: str, queue: str, binding_key: str, dead_letter=False, ): queue_arguments = {} if dead_letter: # Note: If queue is already created with another RABBITMQ_MESSAGE_TTL, it will fail. message_ttl = int(os.environ.get("RABBITMQ_MESSAGE_TTL", "1000")) # 1 second queue_arguments = { "x-message-ttl": message_ttl, "x-dead-letter-exchange": f"dlx-{exchange}", } try: channel.exchange_declare(exchange=exchange, exchange_type="topic", durable=True) result = channel.queue_declare(queue=queue, arguments=queue_arguments, durable=True) channel.queue_bind(exchange=exchange, queue=result.method.queue, routing_key=binding_key) except Exception as error: raise TypeError( f"RabbitMQEventPublisher: Cannot create the exchange ({exchange}) and bind to queue ({queue}) using given routing_key ({binding_key}) \n{error}" )
def gracefully_stop_runner(channel: BlockingChannel, count: int): runners = db.get_collection('corp-hq', 'runners') all_runners = list(runners.find()) for i in range(count): runner = all_runners[i] channel.basic_publish(exchange='', routing_key=runner['name'], body='') LOG.info('Request for runner "%s" to stop sent.', runner['name'])
def callback(ch: BlockingChannel, method, properties, body): msg = str(body, "utf8").split(":") print('# [%s] %s 메세지를 받았습니다. \n %r' % (datetime.datetime.now(), msg[0], body)) time.sleep(int(str(msg[1])) / 10) print(" # [%s] 완료했습니다." % datetime.datetime.now()) ch.basic_ack(delivery_tag=method.delivery_tag)
def do_publish(channel: BlockingChannel) -> None: channel.basic_publish( exchange='', routing_key=queue_name, properties=pika.BasicProperties(delivery_mode=2), body=body) statsd.incr(f"rabbitmq.publish.{queue_name}")
def _uninstrument_channel_functions(channel: BlockingChannel) -> None: for function_name in _FUNCTIONS_TO_UNINSTRUMENT: if not hasattr(channel, function_name): continue function = getattr(channel, function_name) if hasattr(function, "_original_function"): channel.__setattr__(function_name, function._original_function) unwrap(channel, "basic_consume")
def wrapped_consumer( ch: BlockingChannel, method: Basic.Deliver, properties: pika.BasicProperties, body: bytes, ) -> None: callback([orjson.loads(body)]) ch.basic_ack(delivery_tag=method.delivery_tag)
def send_message(channel: BlockingChannel, body: str): """ Send a basic message to the given channel :param channel: The channel to send to. :param body: The body to send. """ print(f"Sending message: {body}") channel.basic_publish(exchange="", routing_key=QUEUE_NAME, body=body)
def on_rpc_response(ch: BlockingChannel, method, props, body): if props.correlation_id == self.corr_id: decoded_body = body.decode() log_info('GET RPC REPLY', queue=reply_queue, correlation_id=self.corr_id, body=decoded_body) self.result = jsons.loads(decoded_body) ch.basic_ack(delivery_tag=method.delivery_tag)
def _on_message( channel: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes, ) -> None: quote: StockQuote = pickle.loads(body) handler([quote]) channel.basic_ack(method.delivery_tag)
def send_message(channel: BlockingChannel, message: str) -> bool: try: channel.basic_publish(exchange='', routing_key=settings.RABBITMQ_QUEUE, body=bytes(message, encoding='utf-8')) return True except Exception as err: print(err) return False
def opened(channel: BlockingChannel) -> None: while True: (meta, _, message) = channel.basic_get(queue_name) if message is None: break channel.basic_ack(meta.delivery_tag) messages.append(message)
def _message_callback(self, ch: BlockingChannel, method: Basic.Deliver, _, body: bytes): try: body_as_dict = RabbitMQBrokerConsumer._decode_body(body) self._callback(body_as_dict) except Exception as e: print(f'Error during handling message {body}: {e}') finally: ch.basic_ack(delivery_tag=method.delivery_tag)
def on_message(self, ch: BlockingChannel, method, properties, body: str): payload = json.loads(body) response = self.callback(self.service, payload) payload = {**payload, **response} self.channel.basic_publish("", self.queue_out, body=json.dumps(payload), mandatory=True) ch.basic_ack(delivery_tag=method.delivery_tag)
def wrapped_consumer(ch: BlockingChannel, method: Basic.Deliver, properties: pika.BasicProperties, body: str) -> None: try: consumer(ch, method, properties, body) ch.basic_ack(delivery_tag=method.delivery_tag) except Exception as e: ch.basic_nack(delivery_tag=method.delivery_tag) raise e
def on_request(self, channel: BlockingChannel, method, props, body): request = dill.loads(body) response = self.execute_request(request) channel.basic_publish(exchange='', routing_key=props.reply_to, properties=pika.BasicProperties(correlation_id = \ props.correlation_id), body=dill.dumps(response)) channel.basic_ack(delivery_tag=method.delivery_tag)
def req_callback(ch: BlockingChannel, method: spec.Basic.Deliver, _: spec.BasicProperties, body: bytes): body = load_object(body) ack = callback(body) if ack is None: ack = True if ack: ch.basic_ack(delivery_tag=method.delivery_tag) else: ch.basic_nack(delivery_tag=method.delivery_tag)
def target_consuming_executor(task: TargetTask, channel: BlockingChannel) -> NoReturn: """ Starts endless consuming messages from RabbitMQ queue :param task: Parameters for producer (routing key and queue label) :param channel: Prepared blocking channel for consuming :return: """ channel.basic_consume(queue=task.queue_label, on_message_callback=nxlog_callback) channel.start_consuming() logger.warning("Should appear only on exception. !!!")
def raw_answer_callback(ch: BlockingChannel, method: spec.Basic.Deliver, _: spec.BasicProperties, body: bytes): body: AnsBody = load_object(body) ack = answer_callback(body['id'], body['ans']) if ack is None: ack = True if ack: ch.basic_ack(delivery_tag=method.delivery_tag) else: ch.basic_nack(delivery_tag=method.delivery_tag)
def ack_message(self, channel: BlockingChannel, delivery_tag: int) -> None: if channel.is_open: logging.info( "%s", f"Message (delivery_tag={delivery_tag}) acknowledged") channel.basic_ack(delivery_tag) else: logging.info( "%s", (f"Message (delivery_tag={delivery_tag}) " "NOT acknowledged (channel closed)"), )
def nack_message(self, channel: BlockingChannel, delivery_tag: int) -> None: if channel.is_open: logging.info("%s", f"Message (delivery_tag={delivery_tag}) rejected") channel.basic_nack(delivery_tag) else: logging.info( "%s", f"Message (delivery_tag={delivery_tag}) NOT rejected (channel closed)", )
def cb(ch: BlockingChannel, method: pika.spec.Basic.Deliver, properties: pika.spec.BasicProperties, body: bytes): key = properties.correlation_id try: callback(ch, method, properties, body) ch.basic_ack(delivery_tag=method.delivery_tag) self.logger.info(f" [x] Message {key} processed!") except Exception: self.logger.info( f" [-] failed to process message {key}: {traceback.format_exc()}" )
def _instrument_basic_publish( channel: BlockingChannel, tracer: Tracer, publish_hook: utils.HookT = utils.dummy_callback, ) -> None: original_function = getattr(channel, "basic_publish") decorated_function = utils._decorate_basic_publish( original_function, channel, tracer, publish_hook) setattr(decorated_function, "_original_function", original_function) channel.__setattr__("basic_publish", decorated_function) channel.basic_publish = decorated_function
def on_rpc_response(ch: BlockingChannel, method, props, body): if props.correlation_id == self.corr_id: decoded_body = body.decode() self.result = jsons.loads(decoded_body) print({ 'action': 'get_rmq_rpc_reply', 'queue': reply_queue, 'correlation_id': self.corr_id, 'result': self.result }) self.replied = True ch.basic_ack(delivery_tag=method.delivery_tag)
def req_callback(ch: BlockingChannel, method: spec.Basic.Deliver, _: spec.BasicProperties, body: bytes): body: ReqBody = load_object(body) ack = request_callback( body['req'], lambda answer: answer_callback(body['id'], answer)) if ack is None: ack = True if ack: ch.basic_ack(delivery_tag=method.delivery_tag) else: ch.basic_nack(delivery_tag=method.delivery_tag)