def thrift_pool_from_config(app_config, prefix, **kwargs):
"""Make a ThriftConnectionPool from a configuration dictionary.
The keys useful to :py:func:`thrift_pool_from_config` should be prefixed,
e.g. ``example_service.endpoint`` etc. The ``prefix`` argument specifies
the prefix used to filter keys. Each key is mapped to a corresponding
keyword argument on the :py:class:`ThriftConnectionPool` constructor. Any
keyword arguments given to this function will be also be passed through to
the constructor. Keyword arguments take precedence over the configuration
file.
Supported keys:
* ``endpoint`` (required): A ``host:port`` pair, e.g. ``localhost:2014``,
where the Thrift server can be found.
* ``size``: The size of the connection pool.
* ``max_age``: The oldest a connection can be before it's recycled and
replaced with a new one. Written as a time span e.g. ``1 minute``.
* ``timeout``: The maximum amount of time a connection attempt or RPC call
can take before a TimeoutError is raised.
* ``max_retries``: The maximum number of times the pool will attempt to
open a connection.
"""
assert prefix.endswith(".")
config_prefix = prefix[:-1]
cfg = config.parse_config(
app_config,
{
config_prefix: {
"endpoint":
config.Endpoint,
"size":
config.Optional(config.Integer, default=10),
"max_age":
config.Optional(config.Timespan,
default=config.Timespan("1 minute")),
"timeout":
config.Optional(config.Timespan,
default=config.Timespan("1 second")),
"max_retries":
config.Optional(config.Integer, default=3),
}
},
)
options = getattr(cfg, config_prefix)
if options.size is not None:
kwargs.setdefault("size", options.size)
if options.max_age is not None:
kwargs.setdefault("max_age", options.max_age.total_seconds())
if options.timeout is not None:
kwargs.setdefault("timeout", options.timeout.total_seconds())
if options.max_retries is not None:
kwargs.setdefault("max_retries", options.max_retries)
return ThriftConnectionPool(endpoint=options.endpoint, **kwargs)
def test_timespan_invalid(self):
with self.assertRaises(ValueError):
config.Timespan("")
with self.assertRaises(ValueError):
config.Timespan("a b")
with self.assertRaises(ValueError):
config.Timespan("10 florgles")
with self.assertRaises(ValueError):
config.Timespan("a b c")
with self.assertRaises(ValueError):
config.Timespan("3.2 hours")
def test_timespan(self):
result = config.Timespan("30 milliseconds")
self.assertAlmostEqual(result.total_seconds(), 0.03)
result = config.Timespan("1 second")
self.assertEqual(result.total_seconds(), 1)
result = config.Timespan("2 seconds")
self.assertEqual(result.total_seconds(), 2)
result = config.Timespan("30 minutes")
self.assertEqual(result.total_seconds(), 1800)
result = config.Timespan("2 hours")
self.assertEqual(result.total_seconds(), 7200)
result = config.Timespan("1 day")
self.assertEqual(result.total_seconds(), 86400)
def tracing_client_from_config(
raw_config: config.RawConfig,
log_if_unconfigured: bool = True) -> tracing.TracingClient:
"""Configure and return a tracing client.
This expects one configuration option and can take many optional ones:
``tracing.service_name``
The name for the service this observer is registered to.
``tracing.endpoint`` (optional)
(Deprecated in favor of the sidecar model.) Destination to record span data.
``tracing.queue_name`` (optional)
Name of POSIX queue where spans are recorded
``tracing.max_span_queue_size`` (optional)
Span processing queue limit.
``tracing.num_span_workers`` (optional)
Number of worker threads for span processing.
``tracing.span_batch_interval`` (optional)
Wait time for span processing in seconds.
``tracing.num_conns`` (optional)
Pool size for remote recorder connection pool.
``tracing.sample_rate`` (optional)
Percentage of unsampled requests to record traces for (e.g. "37%")
:param dict raw_config: The application configuration which should have
settings for the tracing client.
:param bool log_if_unconfigured: When the client is not configured, should
trace spans be logged or discarded silently?
:return: A configured client.
:rtype: :py:class:`baseplate.diagnostics.tracing.TracingClient`
"""
cfg = config.parse_config(
raw_config,
{
"tracing": {
"service_name":
config.String,
"endpoint":
config.Optional(config.Endpoint),
"queue_name":
config.Optional(config.String),
"max_span_queue_size":
config.Optional(config.Integer, default=50000),
"num_span_workers":
config.Optional(config.Integer, default=5),
"span_batch_interval":
config.Optional(config.Timespan,
default=config.Timespan("500 milliseconds")),
"num_conns":
config.Optional(config.Integer, default=100),
"sample_rate":
config.Optional(config.Fallback(config.Percent, config.Float),
default=0.1),
}
},
)
# pylint: disable=maybe-no-member
return tracing.make_client(
service_name=cfg.tracing.service_name,
tracing_endpoint=cfg.tracing.endpoint,
tracing_queue_name=cfg.tracing.queue_name,
max_span_queue_size=cfg.tracing.max_span_queue_size,
num_span_workers=cfg.tracing.num_span_workers,
span_batch_interval=cfg.tracing.span_batch_interval.total_seconds(),
num_conns=cfg.tracing.num_conns,
sample_rate=cfg.tracing.sample_rate,
log_if_unconfigured=log_if_unconfigured,
)
def zookeeper_client_from_config(secrets, app_config, read_only=None):
"""Configure and return a ZooKeeper client.
There are several configuration options:
``zookeeper.hosts``
A comma-delimited list of hosts with optional ``chroot`` at the end.
For example ``zk01:2181,zk02:2181`` or
``zk01:2181,zk02:2181/some/root``.
``zookeeper.credentials``
(Optional) A comma-delimited list of paths to secrets in the secrets
store that contain ZooKeeper authentication credentials. Secrets should
be of the "simple" type and contain ``username:password``.
``zookeeper.timeout``
(Optional) A time span of how long to wait for each connection attempt.
The client will attempt forever to reconnect on connection loss.
:param baseplate.secrets.SecretsStore secrets: A secrets store object
:param dict raw_config: The application configuration which should have
settings for the ZooKeeper client.
:param bool read_only: Whether or not to allow connections to read-only
ZooKeeper servers.
:rtype: :py:class:`kazoo.client.KazooClient`
"""
full_cfg = config.parse_config(
app_config,
{
"zookeeper": {
"hosts":
config.String,
"credentials":
config.Optional(config.TupleOf(config.String), default=[]),
"timeout":
config.Optional(config.Timespan,
default=config.Timespan("5 seconds")),
}
},
)
# pylint: disable=maybe-no-member
cfg = full_cfg.zookeeper
auth_data = []
for path in cfg.credentials:
credentials = secrets.get_simple(path)
auth_data.append(("digest", credentials.decode("utf8")))
return KazooClient(
cfg.hosts,
timeout=cfg.timeout.total_seconds(),
auth_data=auth_data,
read_only=read_only,
# this retry policy tells Kazoo how often it should attempt connections
# to ZooKeeper from its worker thread/greenlet. when the connection is
# lost during normal operation (i.e. after it was first established)
# Kazoo will do retries quietly in the background while the application
# continues forward. because of this, we want it to retry forever so
# that it doesn't just give up at some point. the application can still
# decide if it wants to exit after being disconnected for an amount of
# time by polling the KazooClient.connected property.
#
# note: KazooClient.start() has a timeout parameter which defaults to
# 15 seconds and controls the maximum amount of time start() will block
# waiting for the background thread to confirm it has established a
# connection. so even though we do infinite retries here, users of this
# function can configure the amount of time they are willing to wait
# for initial connection.
connection_retry=dict(
max_tries=-1, # keep reconnecting forever
delay=0.1, # initial delay
backoff=2, # exponential backoff
max_jitter=1, # maximum amount to jitter sleeptimes
max_delay=60, # never wait longer than this
),
)
