Пример #1
0
 def retrying(self, **policy):
     yield self.__class__(
         self.client_cls,
         self.pool,
         self.server_span,
         self.namespace,
         retry_policy=RetryPolicy.new(**policy),
     )
Пример #2
0
    def __init__(self, endpoint, size=10, max_age=120, timeout=1, max_retries=3,
                 protocol_factory=THeaderProtocol.THeaderProtocolFactory()):
        self.endpoint = endpoint
        self.max_age = max_age
        self.retry_policy = RetryPolicy.new(attempts=max_retries)
        self.timeout = timeout
        self.protocol_factory = protocol_factory

        self.pool = queue.LifoQueue()
        for _ in range(size):
            self.pool.put(None)
Пример #3
0
 def __init__(self,
              client_cls,
              pool,
              server_span,
              namespace,
              retry_policy=None):
     self.client_cls = client_cls
     self.pool = pool
     self.server_span = server_span
     self.namespace = namespace
     self.retry_policy = retry_policy or RetryPolicy.new(attempts=1)
Пример #4
0
    def test_attempts_and_backoff(self, sleep):
        policy = RetryPolicy.new(backoff=0.2, attempts=3)
        retries = iter(policy)

        next(retries)

        next(retries)
        sleep.assert_called_with(0.2)

        next(retries)
        sleep.assert_called_with(0.4)

        with self.assertRaises(StopIteration):
            next(retries)
Пример #5
0
    def __init__(self, path, parser, timeout=None, binary=False, encoding=None,
                 newline=None):
        if sys.version_info.major < 3 and encoding is not None:
            raise TypeError("'encoding' keyword argument for FileWatcher() is "
                            "not supported in Python 2")

        if sys.version_info.major < 3 and newline is not None:
            raise TypeError("'newline' keyword argument for FileWatcher() is "
                            "not supported in Python 2")

        if binary and encoding is not None:
            raise TypeError("'encoding' is not supported in binary mode.")

        if binary and newline is not None:
            raise TypeError("'newline' is not supported in binary mode.")

        self._path = path
        self._parser = parser
        self._mtime = 0
        self._data = _NOT_LOADED
        self._mode = "rb" if binary else "r"
        # Since Python 2 does not support these kwargs, we store them as a dict
        # that we `**` in the call to `open` in `get_data` so we do not have to
        # call `open` in different ways depending on the Python version.  This
        # can change if/when Python 2 support is dropped.
        self._open_options = {}

        if encoding:
            self._open_options['encoding'] = encoding

        if newline is not None:
            self._open_options['newline'] = newline

        if timeout is not None:
            last_error = None
            for _ in RetryPolicy.new(budget=timeout, backoff=0.01):
                if self._data is not _NOT_LOADED:
                    break

                try:
                    self.get_data()
                except WatchedFileNotAvailableError as exc:
                    last_error = exc
                else:
                    break

                logging.warning("%s: file not yet available. sleeping.", path)
            else:
                raise WatchedFileNotAvailableError(self._path,
                    "timed out. last error was: %s" % last_error.inner)
Пример #6
0
    def publish(self, payload):
        """Publish spans to Zipkin API.

        :param baseplate.events.publisher.SerializedBatch payload: Count and
            payload to publish.
        """
        if not payload.count:
            return

        logger.info("Sending batch of %d traces", payload.count)
        headers = {
            "User-Agent": "baseplate-trace-publisher/1.0",
            "Content-Type": "application/json",
        }
        for time_remaining in RetryPolicy.new(attempts=self.retry_limit):
            try:
                with self.metrics.timer("post"):
                    response = self.session.post(
                        self.endpoint,
                        data=payload.bytes,
                        headers=headers,
                        timeout=self.post_timeout,
                        stream=False,
                    )
                response.raise_for_status()
            except requests.HTTPError as exc:
                self.metrics.counter("error.http").increment()
                response = getattr(exc, "response", None)
                if response is not None:
                    logger.exception("HTTP Request failed. Error: %s",
                                     response.text)
                    # If client error, crash
                    if response.status_code < 500:
                        raise
                else:
                    logger.exception(
                        "HTTP Request failed. Response not available")
            except IOError:
                self.metrics.counter("error.io").increment()
                logger.exception("HTTP Request failed")
            else:
                self.metrics.counter("sent").increment(payload.count)
                return

        raise MaxRetriesError(
            "ZipkinPublisher exhausted allowance of %d retries." %
            self.retry_limit)
Пример #7
0
    def put(self, message, timeout=None):
        """Add a message to the queue.

        :param float timeout: If the queue is full, the call will block up to
            ``timeout`` seconds or forever if ``None``.
        :raises: :py:exc:`TimedOutError` The queue was full for the allowed
            duration of the call.

        """
        for time_remaining in RetryPolicy.new(budget=timeout):
            try:
                return self.queue.send(message=message)
            except posix_ipc.SignalError:  # pragma: nocover
                continue  # interrupted, just try again
            except posix_ipc.BusyError:
                select.select([], [self.queue.mqd], [], time_remaining)

        raise TimedOutError
Пример #8
0
    def __init__(self,
                 path,
                 parser,
                 timeout=None,
                 binary=False,
                 encoding=None,
                 newline=None):
        if binary and encoding is not None:
            raise TypeError("'encoding' is not supported in binary mode.")

        if binary and newline is not None:
            raise TypeError("'newline' is not supported in binary mode.")

        self._path = path
        self._parser = parser
        self._mtime = 0
        self._data = _NOT_LOADED
        self._mode = "rb" if binary else "r"
        self._open_options = {}

        if encoding:
            self._open_options['encoding'] = encoding

        if newline is not None:
            self._open_options['newline'] = newline

        if timeout is not None:
            last_error = None
            for _ in RetryPolicy.new(budget=timeout, backoff=0.01):
                if self._data is not _NOT_LOADED:
                    break

                try:
                    self.get_data()
                except WatchedFileNotAvailableError as exc:
                    last_error = exc
                else:
                    break

                logging.warning("%s: file not yet available. sleeping.", path)
            else:
                raise WatchedFileNotAvailableError(
                    self._path,
                    "timed out. last error was: %s" % last_error.inner)
Пример #9
0
    def publish(self, payload):
        if not payload.count:
            return

        logger.info("sending batch of %d events", payload.count)
        compressed_payload = gzip_compress(payload.bytes)
        headers = {
            "Date": email.utils.formatdate(usegmt=True),
            "User-Agent": "baseplate-event-publisher/1.0",
            "Content-Type": "application/json",
            "X-Signature": self._sign_payload(payload.bytes),
            "Content-Encoding": "gzip",
        }

        for _ in RetryPolicy.new(budget=MAX_RETRY_TIME, backoff=RETRY_BACKOFF):
            try:
                with self.metrics.timer("post"):
                    response = self.session.post(
                        self.url,
                        headers=headers,
                        data=compressed_payload,
                        timeout=POST_TIMEOUT,
                        # http://docs.python-requests.org/en/latest/user/advanced/#keep-alive
                        stream=False,
                    )
                response.raise_for_status()
            except requests.HTTPError as exc:
                self.metrics.counter("error.http").increment()

                # we should crash if it's our fault
                response = getattr(exc, "response", None)
                if response is not None and response.status_code < 500:
                    logger.exception("HTTP Request failed. Error: %s",
                                     response.text)
                    raise
                logger.exception("HTTP Request failed.")
            except IOError:
                self.metrics.counter("error.io").increment()
                logger.exception("HTTP Request failed")
            else:
                self.metrics.counter("sent").increment(payload.count)
                return

        raise MaxRetriesError("could not sent batch")
Пример #10
0
    def __init__(
        self,
        path: str,
        parser: Callable[[IO], T],
        timeout: Optional[int] = None,
        binary: bool = False,
        encoding: Optional[str] = None,
        newline: Optional[str] = None,
    ):
        if binary and encoding is not None:
            raise TypeError("'encoding' is not supported in binary mode.")

        if binary and newline is not None:
            raise TypeError("'newline' is not supported in binary mode.")

        self._path = path
        self._parser = parser
        self._mtime = 0.0
        self._data: Union[T, Type[_NOT_LOADED]] = _NOT_LOADED
        self._open_options = _OpenOptions(mode="rb" if binary else "r",
                                          encoding=encoding,
                                          newline=newline)

        if timeout is not None:
            last_error = None
            for _ in RetryPolicy.new(budget=timeout, backoff=0.01):
                if self._data is not _NOT_LOADED:
                    break

                try:
                    self.get_data()
                except WatchedFileNotAvailableError as exc:
                    last_error = exc
                else:
                    break

                logging.warning("%s: file not yet available. sleeping.", path)
            else:
                last_error = typing.cast(WatchedFileNotAvailableError,
                                         last_error)
                raise WatchedFileNotAvailableError(
                    self._path,
                    f"timed out. last error was: {last_error.inner}")
Пример #11
0
    def get(self, timeout=None):
        """Read a message from the queue.

        :param float timeout: If the queue is empty, the call will block up to
            ``timeout`` seconds or forever if ``None``.
        :raises: :py:exc:`TimedOutError` The queue was empty for the allowed
            duration of the call.

        """
        for time_remaining in RetryPolicy.new(budget=timeout):
            try:
                message, _ = self.queue.receive()
                return message
            except posix_ipc.SignalError:  # pragma: nocover
                continue  # interrupted, just try again
            except posix_ipc.BusyError:
                select.select([self.queue.mqd], [], [], time_remaining)

        raise TimedOutError
Пример #12
0
    def test_budget_overrides_backoff(self, sleep, time):
        policy = RetryPolicy.new(backoff=0.1, budget=1)

        time.return_value = 0
        retries = iter(policy)
        time_remaining = next(retries)
        self.assertAlmostEqual(time_remaining, 1)
        self.assertEqual(sleep.call_count, 0)

        time.return_value = .5
        time_remaining = next(retries)
        self.assertAlmostEqual(time_remaining, .4)
        sleep.assert_called_with(.1)

        time.return_value = .9
        time_remaining = next(retries)
        self.assertAlmostEqual(time_remaining, 0)
        self.assertAlmostEqual(sleep.call_args[0][0], 0.1, places=2)

        time.return_value = 1
        with self.assertRaises(StopIteration):
            next(retries)
Пример #13
0
    def get_batch(self, max_items, timeout):
        """Return a batch of messages.

        :param int max_items: The maximum batch size.
        :param int timeout: The maximum time to wait in seconds, or ``None``
            for no timeout.

        """
        if timeout == 0:
            block = False
        else:
            block = True

        batch = []
        retry_policy = RetryPolicy.new(attempts=max_items, budget=timeout)
        for time_remaining in retry_policy:
            item = self.worker.get_message(block=block, timeout=time_remaining)
            if item is None:
                break

            batch.append(item)

        return batch
Пример #14
0
    def __init__(self, path, parser, timeout=None):
        self._path = path
        self._parser = parser
        self._mtime = 0
        self._data = _NOT_LOADED

        if timeout is not None:
            last_error = None
            for _ in RetryPolicy.new(budget=timeout, backoff=0.01):
                if self._data is not _NOT_LOADED:
                    break

                try:
                    self.get_data()
                except WatchedFileNotAvailableError as exc:
                    last_error = exc
                else:
                    break

                logging.warning("%s: file not yet available. sleeping.", path)
            else:
                raise WatchedFileNotAvailableError(
                    self._path,
                    "timed out. last error was: %s" % last_error.inner)