def _start_processing(self, connection: pika.BlockingConnection, channel: pika.channel, delivery_tag, body): """ This should be executed within a new thread """ try: if self._parse_func: self._parse_func(json.loads(body)) except Exception as e: self.log.exception("Failed to parse message: " + str(body)) # nack this message connection.add_callback_threadsafe(functools.partial(channel.basic_nack, delivery_tag=delivery_tag)) raise e # Acknowledge the message connection.add_callback_threadsafe(functools.partial(channel.basic_ack, delivery_tag=delivery_tag))
def do_work(conn: pika.BlockingConnection, channel: pika.adapters.blocking_connection.BlockingChannel, method: pika.spec.Basic.Deliver, properties: pika.spec.BasicProperties, body: bytes, custom_func): try: custom_func(json.loads(body), method.routing_key) conn.add_callback_threadsafe( functools.partial(ack_message, channel, method.delivery_tag)) except Exception as e: logger.warning(e) if conn.is_open: conn.add_callback_threadsafe( functools.partial(nack_message, channel, method.delivery_tag)) finally: gc.collect()
def _call_event_handler( handler: Callable, event: DomainEvent, connection: BlockingConnection, acknowledge: Callable, retry: Callable, reject: Callable, max_retries: int, ) -> None: # The handler is executed in a separate worker thread. Handle any errors # and trigger retries, dead-lettering or acknowledgement via threadsafe # callback on the connection. try: handler(event) except Retry as error: if event.retries < max_retries: # Publish manually to the delay exchange with a per-message TTL msg = "Retry ({retries}) consuming event {event} in {delay:.1f}s" log.info( msg.format(event=event, retries=event.retries, delay=error.delay)) delayed_retry = partial(retry, delay=error.delay) connection.add_callback_threadsafe(acknowledge) connection.add_callback_threadsafe(delayed_retry) else: # Reject puts the message into the dead-letter queue if there is # one, otherwise the message is discarded. msg = "Exceeded max retries ({}) for {} event".format( max_retries, event.routing_key) log.error(msg, exc_info=True, extra=event.event_data) connection.add_callback_threadsafe(reject) except: # noqa: E722 # Note: If we want immediate requeueing, add a `RequeueError` # that a consumer can raise to trigger requeuing. Dead-letter # queues are a better choice in most cases. connection.add_callback_threadsafe(reject) log.exception("Event has been dead-lettered or discarded") else: connection.add_callback_threadsafe(acknowledge)
def _heartbeat(connection: pika.BlockingConnection): while True: logger.debug('SENDING HEARTBEAT') connection.add_callback_threadsafe( lambda: connection.process_data_events(time_limit=10)) time.sleep(10)
class Client: server_queue: str timeout: int connection_parameters: Parameters _thread: Optional[Thread] = None conn: Optional[BlockingConnection] = None channel: Optional[BlockingChannel] = None _waiting: Dict[str, Future] = field(default_factory=dict) def worker(self) -> None: assert self.channel time.sleep(1.5) logger.debug('starting worker listening on %s', self.server_queue) try: self.channel.start_consuming() except Exception as e: logger.exception('worker died') self.connect() def connect(self) -> 'Client': self.conn = BlockingConnection(self.connection_parameters) self.channel = self.conn.channel() self._thread = t = Thread(target=self.worker) t.daemon = True t.start() self.channel.basic_consume(REPLY_TO, self.recieve, auto_ack=True) return self def call(self, method: str, *args: Any, **kwargs: Any) -> T: assert self.conn and self.channel f: Future = Future() key = uuid4().hex self._waiting[key] = f self.conn.add_callback_threadsafe(lambda: self.channel.basic_publish( exchange="", routing_key=self.server_queue, body=dill.dumps({ "method": method, "args": args, "kwargs": kwargs, "key": key }), properties=BasicProperties(reply_to=REPLY_TO), )) return f.result(timeout=self.timeout) def __getattr__(self, method: str) -> Callable: return partial(self.call, method) def recieve( self, ch: BlockingChannel, method_frame: Method, properties: BasicProperties, _body: str, ) -> None: body = dill.loads(_body) f = self._waiting.pop(body["key"]) if "body" in body: f.set_result(body["body"]) elif "exception" in body: f.set_exception(body["exception"]) else: raise RuntimeError()