def thrift_pool_from_config(app_config, prefix, **kwargs): """Make a ThriftConnectionPool from a configuration dictionary. The keys useful to :py:func:`thrift_pool_from_config` should be prefixed, e.g. ``example_service.endpoint`` etc. The ``prefix`` argument specifies the prefix used to filter keys. Each key is mapped to a corresponding keyword argument on the :py:class:`ThriftConnectionPool` constructor. Any keyword arguments given to this function will be also be passed through to the constructor. Keyword arguments take precedence over the configuration file. Supported keys: * ``endpoint`` (required): A ``host:port`` pair, e.g. ``localhost:2014``, where the Thrift server can be found. * ``size``: The size of the connection pool. * ``max_age``: The oldest a connection can be before it's recycled and replaced with a new one. Written as a time span e.g. ``1 minute``. * ``timeout``: The maximum amount of time a connection attempt or RPC call can take before a TimeoutError is raised. * ``max_retries``: The maximum number of times the pool will attempt to open a connection. """ assert prefix.endswith(".") config_prefix = prefix[:-1] cfg = config.parse_config( app_config, { config_prefix: { "endpoint": config.Endpoint, "size": config.Optional(config.Integer, default=10), "max_age": config.Optional(config.Timespan, default=config.Timespan("1 minute")), "timeout": config.Optional(config.Timespan, default=config.Timespan("1 second")), "max_retries": config.Optional(config.Integer, default=3), } }, ) options = getattr(cfg, config_prefix) if options.size is not None: kwargs.setdefault("size", options.size) if options.max_age is not None: kwargs.setdefault("max_age", options.max_age.total_seconds()) if options.timeout is not None: kwargs.setdefault("timeout", options.timeout.total_seconds()) if options.max_retries is not None: kwargs.setdefault("max_retries", options.max_retries) return ThriftConnectionPool(endpoint=options.endpoint, **kwargs)
def test_timespan_invalid(self): with self.assertRaises(ValueError): config.Timespan("") with self.assertRaises(ValueError): config.Timespan("a b") with self.assertRaises(ValueError): config.Timespan("10 florgles") with self.assertRaises(ValueError): config.Timespan("a b c") with self.assertRaises(ValueError): config.Timespan("3.2 hours")
def test_timespan(self): result = config.Timespan("30 milliseconds") self.assertAlmostEqual(result.total_seconds(), 0.03) result = config.Timespan("1 second") self.assertEqual(result.total_seconds(), 1) result = config.Timespan("2 seconds") self.assertEqual(result.total_seconds(), 2) result = config.Timespan("30 minutes") self.assertEqual(result.total_seconds(), 1800) result = config.Timespan("2 hours") self.assertEqual(result.total_seconds(), 7200) result = config.Timespan("1 day") self.assertEqual(result.total_seconds(), 86400)
def tracing_client_from_config( raw_config: config.RawConfig, log_if_unconfigured: bool = True) -> tracing.TracingClient: """Configure and return a tracing client. This expects one configuration option and can take many optional ones: ``tracing.service_name`` The name for the service this observer is registered to. ``tracing.endpoint`` (optional) (Deprecated in favor of the sidecar model.) Destination to record span data. ``tracing.queue_name`` (optional) Name of POSIX queue where spans are recorded ``tracing.max_span_queue_size`` (optional) Span processing queue limit. ``tracing.num_span_workers`` (optional) Number of worker threads for span processing. ``tracing.span_batch_interval`` (optional) Wait time for span processing in seconds. ``tracing.num_conns`` (optional) Pool size for remote recorder connection pool. ``tracing.sample_rate`` (optional) Percentage of unsampled requests to record traces for (e.g. "37%") :param dict raw_config: The application configuration which should have settings for the tracing client. :param bool log_if_unconfigured: When the client is not configured, should trace spans be logged or discarded silently? :return: A configured client. :rtype: :py:class:`baseplate.diagnostics.tracing.TracingClient` """ cfg = config.parse_config( raw_config, { "tracing": { "service_name": config.String, "endpoint": config.Optional(config.Endpoint), "queue_name": config.Optional(config.String), "max_span_queue_size": config.Optional(config.Integer, default=50000), "num_span_workers": config.Optional(config.Integer, default=5), "span_batch_interval": config.Optional(config.Timespan, default=config.Timespan("500 milliseconds")), "num_conns": config.Optional(config.Integer, default=100), "sample_rate": config.Optional(config.Fallback(config.Percent, config.Float), default=0.1), } }, ) # pylint: disable=maybe-no-member return tracing.make_client( service_name=cfg.tracing.service_name, tracing_endpoint=cfg.tracing.endpoint, tracing_queue_name=cfg.tracing.queue_name, max_span_queue_size=cfg.tracing.max_span_queue_size, num_span_workers=cfg.tracing.num_span_workers, span_batch_interval=cfg.tracing.span_batch_interval.total_seconds(), num_conns=cfg.tracing.num_conns, sample_rate=cfg.tracing.sample_rate, log_if_unconfigured=log_if_unconfigured, )
def zookeeper_client_from_config(secrets, app_config, read_only=None): """Configure and return a ZooKeeper client. There are several configuration options: ``zookeeper.hosts`` A comma-delimited list of hosts with optional ``chroot`` at the end. For example ``zk01:2181,zk02:2181`` or ``zk01:2181,zk02:2181/some/root``. ``zookeeper.credentials`` (Optional) A comma-delimited list of paths to secrets in the secrets store that contain ZooKeeper authentication credentials. Secrets should be of the "simple" type and contain ``username:password``. ``zookeeper.timeout`` (Optional) A time span of how long to wait for each connection attempt. The client will attempt forever to reconnect on connection loss. :param baseplate.secrets.SecretsStore secrets: A secrets store object :param dict raw_config: The application configuration which should have settings for the ZooKeeper client. :param bool read_only: Whether or not to allow connections to read-only ZooKeeper servers. :rtype: :py:class:`kazoo.client.KazooClient` """ full_cfg = config.parse_config( app_config, { "zookeeper": { "hosts": config.String, "credentials": config.Optional(config.TupleOf(config.String), default=[]), "timeout": config.Optional(config.Timespan, default=config.Timespan("5 seconds")), } }, ) # pylint: disable=maybe-no-member cfg = full_cfg.zookeeper auth_data = [] for path in cfg.credentials: credentials = secrets.get_simple(path) auth_data.append(("digest", credentials.decode("utf8"))) return KazooClient( cfg.hosts, timeout=cfg.timeout.total_seconds(), auth_data=auth_data, read_only=read_only, # this retry policy tells Kazoo how often it should attempt connections # to ZooKeeper from its worker thread/greenlet. when the connection is # lost during normal operation (i.e. after it was first established) # Kazoo will do retries quietly in the background while the application # continues forward. because of this, we want it to retry forever so # that it doesn't just give up at some point. the application can still # decide if it wants to exit after being disconnected for an amount of # time by polling the KazooClient.connected property. # # note: KazooClient.start() has a timeout parameter which defaults to # 15 seconds and controls the maximum amount of time start() will block # waiting for the background thread to confirm it has established a # connection. so even though we do infinite retries here, users of this # function can configure the amount of time they are willing to wait # for initial connection. connection_retry=dict( max_tries=-1, # keep reconnecting forever delay=0.1, # initial delay backoff=2, # exponential backoff max_jitter=1, # maximum amount to jitter sleeptimes max_delay=60, # never wait longer than this ), )