def do_consume(channel: BlockingChannel) -> None: events: List[Dict[str, Any]] = [] last_process = time.time() max_processed: Optional[int] = None self.is_consuming = True # This iterator technique will iteratively collect up to # batch_size events from the RabbitMQ queue (if present) # before calling the callback with the batch. If not # enough events are present, it will sleep for at most # timeout seconds before calling the callback with the # batch of events it has. for method, properties, body in channel.consume( queue_name, inactivity_timeout=timeout): if body is not None: assert method is not None events.append(orjson.loads(body)) max_processed = method.delivery_tag now = time.time() if len(events) >= batch_size or ( timeout and now >= last_process + timeout): if events: assert max_processed is not None try: callback(events) channel.basic_ack(max_processed, multiple=True) except BaseException: channel.basic_nack(max_processed, multiple=True) raise events = [] last_process = now if not self.is_consuming: break
def ngram_create_consumer(ch: BlockingChannel, method: Basic.Deliver, properties, body: bytes): logger.debug("Recebido: " + body.decode(), extra={"received_args": body}) try: task_info = json.loads(body.decode()) logger.debug("Recuperando tarefa: " + task_info["task"]) ngram_create_task: NGramsCreateTaskModel = NGramsCreateTaskModel.objects(id=task_info["task"]).first() if ngram_create_task is None: raise Exception("Não foi encontrada nenhuma tarefa com o id " + task_info["task"]) except Exception as e: logger.error( "Erro ao recuperar a tarefa", exc_info=True, extra={"received_args": body} ) ch.basic_nack( delivery_tag=method.delivery_tag, requeue=False ) return False success_task = process_task(ngram_create_task) logger.debug("Finalizando tarefa: " + task_info["task"]) if success_task: ch.basic_ack(delivery_tag=method.delivery_tag) return True else: ch.basic_nack( delivery_tag=method.delivery_tag, requeue=False ) return False
def callback(self, ch: BlockingChannel, method: spec.Basic.Deliver, properties: spec.BasicProperties, body: bytes): log.debug(' -> Received ch: %s meth: %s props: %s body: %s', ch, method, properties, body) b = body.decode() try: # General format: {req_id, action, host} act = json.loads(b) # type: dict log.debug(' -> Decoded to %s', act) if 'action' not in act: raise ValueError('"action" was not in decoded json object...') if act['action'] not in self.ACTIONS: raise ValueError('Action "{}" is not a valid action...'.format( act['action'])) except (JSONDecodeError, TypeError, ValueError, AttributeError): log.exception('Error decoding json for %s', b) return ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) try: run_act = self.ACTIONS[act['action']](**act) if run_act: log.debug('Acknowledging success to MQ') return ch.basic_ack(delivery_tag=method.delivery_tag) else: log.debug('Acknowledging failure (try later) to MQ') return ch.basic_nack(delivery_tag=method.delivery_tag) except (InvalidHostException, AttributeError, ValueError) as e: log.warning('Invalid host... Type: %s Msg: %s', type(e), str(e)) return ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) except Exception: log.exception('Unknown exception while handling action call...') return ch.basic_nack(delivery_tag=method.delivery_tag)
def wrapped_consumer(ch: BlockingChannel, method: Basic.Deliver, properties: pika.BasicProperties, body: str) -> None: try: consumer(ch, method, properties, body) ch.basic_ack(delivery_tag=method.delivery_tag) except Exception as e: ch.basic_nack(delivery_tag=method.delivery_tag) raise e
def req_callback(ch: BlockingChannel, method: spec.Basic.Deliver, _: spec.BasicProperties, body: bytes): body = load_object(body) ack = callback(body) if ack is None: ack = True if ack: ch.basic_ack(delivery_tag=method.delivery_tag) else: ch.basic_nack(delivery_tag=method.delivery_tag)
def raw_answer_callback(ch: BlockingChannel, method: spec.Basic.Deliver, _: spec.BasicProperties, body: bytes): body: AnsBody = load_object(body) ack = answer_callback(body['id'], body['ans']) if ack is None: ack = True if ack: ch.basic_ack(delivery_tag=method.delivery_tag) else: ch.basic_nack(delivery_tag=method.delivery_tag)
def nack_message(self, channel: BlockingChannel, delivery_tag: int) -> None: if channel.is_open: logging.info("%s", f"Message (delivery_tag={delivery_tag}) rejected") channel.basic_nack(delivery_tag) else: logging.info( "%s", f"Message (delivery_tag={delivery_tag}) NOT rejected (channel closed)", )
def req_callback(ch: BlockingChannel, method: spec.Basic.Deliver, _: spec.BasicProperties, body: bytes): body: ReqBody = load_object(body) ack = request_callback( body['req'], lambda answer: answer_callback(body['id'], answer)) if ack is None: ack = True if ack: ch.basic_ack(delivery_tag=method.delivery_tag) else: ch.basic_nack(delivery_tag=method.delivery_tag)
def nack_simulation(self, ch: BlockingChannel, method: Basic.Deliver) -> bool: routing_key = method.routing_key if method else None if (self.percentage_simulate_nack is None) or ( random.random() > self.percentage_simulate_nack ): return False if ( self.protected_routing_keys is None or routing_key not in self.protected_routing_keys ): ch.basic_nack(delivery_tag=method.delivery_tag) return True else: return False
def ner_resume_create_consumer(ch: BlockingChannel, method: Basic.Deliver, properties, body: bytes): logger.debug("Recebido: " + body.decode(), extra={"received_args": body}) try: task_info = json.loads(body.decode()) logger.debug("Recuperando tarefa: " + task_info["task"]) ner_resume_create_task: NerResumeCreateTaskModel = NerResumeCreateTaskModel.objects( id=task_info["task"]).first() if ner_resume_create_task is None: raise Exception("Não foi encontrada nenhuma tarefa com o id " + task_info["task"]) except Exception as e: logger.error("Erro ao recuperar a tarefa", exc_info=True, extra={"received_args": body}) ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) return False try: if ner_resume_create_task.status == "success" or \ ner_resume_create_task.progress == ner_resume_create_task.total: ner_resume_create_task.status = "success" ner_resume_create_task.save() success_task = True else: success_task = process_task(ner_resume_create_task) logger.debug("Finalizando tarefa: " + task_info["task"]) if success_task: ch.basic_ack(delivery_tag=method.delivery_tag) return True except ChannelWrongStateError: return False except Exception: ner_resume_create_task.status = "error" ner_resume_create_task.save() logger.error( ner_resume_create_task.error, exc_info=True, extra={"received_args": ner_resume_create_task.to_mongo()}) ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) return False
def rabbitmq_consumer( ch: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes, ): self.printer.print_received_message(method, properties, body) if self.chaos.nack_simulation(ch, method): self.consumer_logger.log_nack_simulation( method, properties, body, handler) return else: self.consumer_logger.log(method, properties, body, handler, log_activity="received_message") try: event = Event.from_json(body) except TypeError: event = Event.from_deprecated_json(body) except Exception as e: self.consumer_logger.log_parser_error(method, properties, body, handler, e) ch.basic_nack(delivery_tag=method.delivery_tag) return self.chaos.delay() if self.chaos.failure_simulation(method): self.consumer_logger.log_failure_simulation( method, properties, body, handler) result = Failure(EventChaosError()) else: params = inspect.getfullargspec(handler).args if "event_bus" in params: connector = RabbitMqConsumerConnector(ch) event_bus = RabbitMqConsumerEventBus( connector, self.organization, self.service) result = handler(event, event_bus) else: result = handler(event) self.printer.print_context(handler, result) if result is None: raise RabbitMqEventConsumerReturnError(handler) derived_action = ConsumerDerivedAction() if result.is_failure: if not properties.headers: properties.headers = { "queue": f"{method.routing_key}.{handler.__name__}" } derived_action = self.handle_consumption_error( ch, method, properties, body, is_store) else: ch.basic_ack(delivery_tag=method.delivery_tag) self.consumer_logger.log( method, properties, body, handler, "computed_message", result, derived_action, ) self.printer.print_separator()