示例#1
0
    def _start_processing(self, connection: pika.BlockingConnection, channel: pika.channel, delivery_tag, body):
        """
        This should be executed within a new thread
        """
        try:
            if self._parse_func:
                self._parse_func(json.loads(body))
        except Exception as e:
            self.log.exception("Failed to parse message: " + str(body))
            # nack this message
            connection.add_callback_threadsafe(functools.partial(channel.basic_nack, delivery_tag=delivery_tag))
            raise e

        # Acknowledge the message
        connection.add_callback_threadsafe(functools.partial(channel.basic_ack, delivery_tag=delivery_tag))
示例#2
0
def do_work(conn: pika.BlockingConnection,
            channel: pika.adapters.blocking_connection.BlockingChannel,
            method: pika.spec.Basic.Deliver,
            properties: pika.spec.BasicProperties, body: bytes, custom_func):
    try:
        custom_func(json.loads(body), method.routing_key)
        conn.add_callback_threadsafe(
            functools.partial(ack_message, channel, method.delivery_tag))
    except Exception as e:
        logger.warning(e)
        if conn.is_open:
            conn.add_callback_threadsafe(
                functools.partial(nack_message, channel, method.delivery_tag))
    finally:
        gc.collect()
示例#3
0
def _call_event_handler(
    handler: Callable,
    event: DomainEvent,
    connection: BlockingConnection,
    acknowledge: Callable,
    retry: Callable,
    reject: Callable,
    max_retries: int,
) -> None:
    # The handler is executed in a separate worker thread. Handle any errors
    # and trigger retries, dead-lettering or acknowledgement via threadsafe
    # callback on the connection.
    try:
        handler(event)
    except Retry as error:
        if event.retries < max_retries:
            # Publish manually to the delay exchange with a per-message TTL
            msg = "Retry ({retries}) consuming event {event} in {delay:.1f}s"
            log.info(
                msg.format(event=event,
                           retries=event.retries,
                           delay=error.delay))
            delayed_retry = partial(retry, delay=error.delay)
            connection.add_callback_threadsafe(acknowledge)
            connection.add_callback_threadsafe(delayed_retry)
        else:
            # Reject puts the message into the dead-letter queue if there is
            # one, otherwise the message is discarded.
            msg = "Exceeded max retries ({}) for {} event".format(
                max_retries, event.routing_key)
            log.error(msg, exc_info=True, extra=event.event_data)
            connection.add_callback_threadsafe(reject)
    except:  # noqa: E722
        # Note: If we want immediate requeueing, add a `RequeueError`
        # that a consumer can raise to trigger requeuing. Dead-letter
        # queues are a better choice in most cases.
        connection.add_callback_threadsafe(reject)
        log.exception("Event has been dead-lettered or discarded")
    else:
        connection.add_callback_threadsafe(acknowledge)
示例#4
0
def _heartbeat(connection: pika.BlockingConnection):
    while True:
        logger.debug('SENDING HEARTBEAT')
        connection.add_callback_threadsafe(
            lambda: connection.process_data_events(time_limit=10))
        time.sleep(10)
示例#5
0
class Client:
    server_queue: str
    timeout: int
    connection_parameters: Parameters
    _thread: Optional[Thread] = None
    conn: Optional[BlockingConnection] = None
    channel: Optional[BlockingChannel] = None
    _waiting: Dict[str, Future] = field(default_factory=dict)

    def worker(self) -> None:
        assert self.channel

        time.sleep(1.5)
        logger.debug('starting worker listening on %s', self.server_queue)
        try:
            self.channel.start_consuming()
        except Exception as e:
            logger.exception('worker died')
            self.connect()

    def connect(self) -> 'Client':
        self.conn = BlockingConnection(self.connection_parameters)
        self.channel = self.conn.channel()

        self._thread = t = Thread(target=self.worker)
        t.daemon = True
        t.start()

        self.channel.basic_consume(REPLY_TO, self.recieve, auto_ack=True)

        return self

    def call(self, method: str, *args: Any, **kwargs: Any) -> T:
        assert self.conn and self.channel

        f: Future = Future()
        key = uuid4().hex
        self._waiting[key] = f
        self.conn.add_callback_threadsafe(lambda: self.channel.basic_publish(
            exchange="",
            routing_key=self.server_queue,
            body=dill.dumps({
                "method": method,
                "args": args,
                "kwargs": kwargs,
                "key": key
            }),
            properties=BasicProperties(reply_to=REPLY_TO),
        ))
        return f.result(timeout=self.timeout)

    def __getattr__(self, method: str) -> Callable:
        return partial(self.call, method)

    def recieve(
        self,
        ch: BlockingChannel,
        method_frame: Method,
        properties: BasicProperties,
        _body: str,
    ) -> None:
        body = dill.loads(_body)
        f = self._waiting.pop(body["key"])
        if "body" in body:
            f.set_result(body["body"])
        elif "exception" in body:
            f.set_exception(body["exception"])
        else:
            raise RuntimeError()