def thrift_pool_from_config(app_config: config.RawConfig, prefix: str,
**kwargs: Any) -> "ThriftConnectionPool":
"""Make a ThriftConnectionPool from a configuration dictionary.
The keys useful to :py:func:`thrift_pool_from_config` should be prefixed,
e.g. ``example_service.endpoint`` etc. The ``prefix`` argument specifies
the prefix used to filter keys. Each key is mapped to a corresponding
keyword argument on the :py:class:`ThriftConnectionPool` constructor. Any
keyword arguments given to this function will be also be passed through to
the constructor. Keyword arguments take precedence over the configuration
file.
Supported keys:
* ``endpoint`` (required): A ``host:port`` pair, e.g. ``localhost:2014``,
where the Thrift server can be found.
* ``size``: The size of the connection pool.
* ``max_age``: The oldest a connection can be before it's recycled and
replaced with a new one. Written as a
:py:func:`~baseplate.lib.config.Timespan` e.g. ``1 minute``.
* ``timeout``: The maximum amount of time a connection attempt or RPC call
can take before a TimeoutError is raised.
(:py:func:`~baseplate.lib.config.Timespan`)
* ``max_connection_attempts``: The maximum number of times the pool will attempt to
open a connection.
.. versionchanged:: 1.2
``max_retries`` was renamed ``max_connection_attempts``.
"""
assert prefix.endswith(".")
parser = config.SpecParser({
"endpoint":
config.Endpoint,
"size":
config.Optional(config.Integer, default=10),
"max_age":
config.Optional(config.Timespan, default=config.Timespan("1 minute")),
"timeout":
config.Optional(config.Timespan, default=config.Timespan("1 second")),
"max_connection_attempts":
config.Optional(config.Integer),
"max_retries":
config.Optional(config.Integer),
})
options = parser.parse(prefix[:-1], app_config)
if options.size is not None:
kwargs.setdefault("size", options.size)
if options.max_age is not None:
kwargs.setdefault("max_age", options.max_age.total_seconds())
if options.timeout is not None:
kwargs.setdefault("timeout", options.timeout.total_seconds())
if options.max_connection_attempts is not None:
kwargs.setdefault("max_connection_attempts",
options.max_connection_attempts)
if options.max_retries is not None:
raise Exception("max_retries was renamed to max_connection_attempts")
return ThriftConnectionPool(endpoint=options.endpoint, **kwargs)
def hvac_factory_from_config(app_config: config.RawConfig,
secrets_store: SecretsStore,
prefix: str = "vault.") -> "HvacContextFactory":
"""Make an HVAC client factory from a configuration dictionary.
The keys useful to :py:func:`hvac_factory_from_config` should be prefixed,
e.g. ``vault.timeout``. The ``prefix`` argument specifies the prefix used
to filter keys.
Supported keys:
* ``timeout``: How long to wait for calls to Vault.
(:py:func:`~baseplate.lib.config.Timespan`)
:param app_config: The raw application configuration.
:param secrets_store: A configured secrets store from which we can get a
Vault authentication token.
:param prefix: The prefix for configuration keys.
"""
assert prefix.endswith(".")
parser = config.SpecParser({
"timeout":
config.Optional(config.Timespan, default=datetime.timedelta(seconds=1))
})
options = parser.parse(prefix[:-1], app_config)
return HvacContextFactory(secrets_store, options.timeout)
def connection_from_config(app_config: config.RawConfig, prefix: str,
**kwargs: Any) -> Connection:
"""Make a Connection from a configuration dictionary.
The keys useful to :py:func:`connection_from_config` should be prefixed,
e.g. ``amqp.hostname`` etc. The ``prefix`` argument specifies the
prefix used to filter keys. Each key is mapped to a corresponding keyword
argument on the :py:class:`~kombu.connection.Connection` constructor. Any
keyword arguments given to this function will be passed through to the
:py:class:`~kombu.connection.Connection` constructor. Keyword arguments
take precedence over the configuration file.
Supported keys:
* ``hostname``
* ``virtual_host``
"""
assert prefix.endswith(".")
parser = config.SpecParser({
"hostname": config.String,
"virtual_host": config.Optional(config.String)
})
options = parser.parse(prefix[:-1], app_config)
return Connection(hostname=options.hostname,
virtual_host=options.virtual_host,
**kwargs)
def exchange_from_config(app_config: config.RawConfig, prefix: str,
**kwargs: Any) -> Exchange:
"""Make an Exchange from a configuration dictionary.
The keys useful to :py:func:`exchange_from_config` should be prefixed,
e.g. ``amqp.exchange_name`` etc. The ``prefix`` argument specifies the
prefix used to filter keys. Each key is mapped to a corresponding keyword
argument on the :py:class:`~kombu.Exchange` constructor. Any keyword
arguments given to this function will be passed through to the
:py:class:`~kombu.Exchange` constructor. Keyword arguments take precedence
over the configuration file.
Supported keys:
* ``exchange_name``
* ``exchange_type``
"""
assert prefix.endswith(".")
parser = config.SpecParser({
"exchange_name": config.Optional(config.String),
"exchange_type": config.String
})
options = parser.parse(prefix[:-1], app_config)
return Exchange(name=options.exchange_name or "",
type=options.exchange_type,
**kwargs)
def engine_from_config(
app_config: config.RawConfig,
secrets: Optional[SecretsStore] = None,
prefix: str = "database.",
**kwargs: Any,
) -> Engine:
"""Make an :py:class:`~sqlalchemy.engine.Engine` from a configuration dictionary.
The keys useful to :py:func:`engine_from_config` should be prefixed, e.g.
``database.url``, etc. The ``prefix`` argument specifies the prefix used to
filter keys.
Supported keys:
* ``url``: the connection URL to the database, passed to
:py:func:`~sqlalchemy.engine.url.make_url` to create the
:py:class:`~sqlalchemy.engine.url.URL` used to connect to the database.
* ``credentials_secret`` (optional): the key used to retrieve the database
credentials from ``secrets`` as a :py:class:`~baseplate.lib.secrets.CredentialSecret`.
If this is supplied, any credentials given in ``url`` we be replaced by
these.
* ``pool_recycle`` (optional): this setting causes the pool to recycle connections after
the given number of seconds has passed. It defaults to -1, or no timeout.
* ``pool_pre_ping`` (optional): when set to true, this setting causes
sqlalchemy to perform a liveness-check query each time a connection is
checked out of the pool. If the liveness-check fails, the connection
is gracefully recycled. This ensures severed connections are handled
more gracefully, at the cost of doing a `SELECT 1` at the start of each
checkout. When used, this obviates most of the reasons you might use
pool_recycle, and as such they shouldn't normally be used
simultaneously. Requires SQLAlchemy 1.3.
"""
assert prefix.endswith(".")
parser = config.SpecParser(
{
"url": config.String,
"credentials_secret": config.Optional(config.String),
"pool_recycle": config.Optional(config.Integer),
"pool_pre_ping": config.Optional(config.Boolean),
}
)
options = parser.parse(prefix[:-1], app_config)
url = make_url(options.url)
if options.pool_recycle is not None:
kwargs.setdefault("pool_recycle", options.pool_recycle)
if options.pool_pre_ping is not None:
kwargs.setdefault("pool_pre_ping", options.pool_pre_ping)
if options.credentials_secret:
if not secrets:
raise TypeError("'secrets' is required if 'credentials_secret' is set")
credentials = secrets.get_credentials(options.credentials_secret)
url.username = credentials.username
url.password = credentials.password
return create_engine(url, **kwargs)
def cluster_from_config(
app_config: config.RawConfig,
secrets: Optional[SecretsStore] = None,
prefix: str = "cassandra.",
execution_profiles: Optional[Dict[str, ExecutionProfile]] = None,
**kwargs: Any,
) -> Cluster:
"""Make a Cluster from a configuration dictionary.
The keys useful to :py:func:`cluster_from_config` should be prefixed, e.g.
``cassandra.contact_points`` etc. The ``prefix`` argument specifies the
prefix used to filter keys. Each key is mapped to a corresponding keyword
argument on the :py:class:`~cassandra.cluster.Cluster` constructor. Any
keyword arguments given to this function will be passed through to the
:py:class:`~cassandra.cluster.Cluster` constructor. Keyword arguments take
precedence over the configuration file.
Supported keys:
* ``contact_points`` (required): comma delimited list of contact points to
try connecting for cluster discovery
* ``port``: The server-side port to open connections to.
* ``credentials_secret`` (optional): the key used to retrieve the database
credentials from ``secrets`` as a :py:class:`~baseplate.lib.secrets.CredentialSecret`.
:param execution_profiles: Configured execution profiles to provide to the
rest of the application.
"""
assert prefix.endswith(".")
parser = config.SpecParser({
"contact_points":
config.TupleOf(config.String),
"port":
config.Optional(config.Integer, default=None),
"credentials_secret":
config.Optional(config.String),
})
options = parser.parse(prefix[:-1], app_config)
if options.port:
kwargs.setdefault("port", options.port)
if options.credentials_secret:
if not secrets:
raise TypeError(
"'secrets' is required if 'credentials_secret' is set")
credentials = secrets.get_credentials(options.credentials_secret)
kwargs.setdefault(
"auth_provider",
PlainTextAuthProvider(username=credentials.username,
password=credentials.password),
)
return Cluster(options.contact_points,
execution_profiles=execution_profiles,
**kwargs)
def engine_from_config(
app_config: config.RawConfig,
secrets: Optional[SecretsStore] = None,
prefix: str = "database.",
**kwargs: Any,
) -> Engine:
"""Make an :py:class:`~sqlalchemy.engine.Engine` from a configuration dictionary.
The keys useful to :py:func:`engine_from_config` should be prefixed, e.g.
``database.url``, etc. The ``prefix`` argument specifies the prefix used to
filter keys.
Supported keys:
* ``url``: the connection URL to the database, passed to
:py:func:`~sqlalchemy.engine.url.make_url` to create the
:py:class:`~sqlalchemy.engine.url.URL` used to connect to the database.
* ``credentials_secret`` (optional): the key used to retrieve the database
credentials from ``secrets`` as a :py:class:`~baseplate.lib.secrets.CredentialSecret`.
If this is supplied, any credentials given in ``url`` we be replaced by
these.
* ``pool_recycle`` (optional): this setting causes the pool to recycle connections after
the given number of seconds has passed. It defaults to -1, or no timeout.
"""
assert prefix.endswith(".")
parser = config.SpecParser({
"url":
config.String,
"credentials_secret":
config.Optional(config.String),
"pool_recycle":
config.Optional(config.Integer),
})
options = parser.parse(prefix[:-1], app_config)
url = make_url(options.url)
if options.pool_recycle is not None:
kwargs.setdefault("pool_recycle", options.pool_recycle)
if options.credentials_secret:
if not secrets:
raise TypeError(
"'secrets' is required if 'credentials_secret' is set")
credentials = secrets.get_credentials(options.credentials_secret)
url.username = credentials.username
url.password = credentials.password
return create_engine(url, **kwargs)
def pool_from_config(app_config: config.RawConfig,
prefix: str = "redis.",
**kwargs: Any) -> redis.ConnectionPool:
"""Make a ConnectionPool from a configuration dictionary.
The keys useful to :py:func:`pool_from_config` should be prefixed, e.g.
``redis.url``, ``redis.max_connections``, etc. The ``prefix`` argument
specifies the prefix used to filter keys. Each key is mapped to a
corresponding keyword argument on the :py:class:`redis.ConnectionPool`
constructor.
Supported keys:
* ``url`` (required): a URL like ``redis://localhost/0``.
* ``max_connections``: an integer maximum number of connections in the pool
* ``socket_connect_timeout``: how long to wait for sockets to connect. e.g.
``200 milliseconds`` (:py:func:`~baseplate.lib.config.Timespan`)
* ``socket_timeout``: how long to wait for socket operations, e.g.
``200 milliseconds`` (:py:func:`~baseplate.lib.config.Timespan`)
"""
assert prefix.endswith(".")
parser = config.SpecParser({
"url":
config.String,
"max_connections":
config.Optional(config.Integer, default=None),
"socket_connect_timeout":
config.Optional(config.Timespan, default=None),
"socket_timeout":
config.Optional(config.Timespan, default=None),
})
options = parser.parse(prefix[:-1], app_config)
if options.max_connections is not None:
kwargs.setdefault("max_connections", options.max_connections)
if options.socket_connect_timeout is not None:
kwargs.setdefault("socket_connect_timeout",
options.socket_connect_timeout.total_seconds())
if options.socket_timeout is not None:
kwargs.setdefault("socket_timeout",
options.socket_timeout.total_seconds())
return redis.BlockingConnectionPool.from_url(options.url, **kwargs)
def connection_from_config(app_config: config.RawConfig,
prefix: str,
secrets: Optional[SecretsStore] = None,
**kwargs: Any) -> Connection:
"""Make a Connection from a configuration dictionary.
The keys useful to :py:func:`connection_from_config` should be prefixed,
e.g. ``amqp.hostname`` etc. The ``prefix`` argument specifies the
prefix used to filter keys. Each key is mapped to a corresponding keyword
argument on the :py:class:`~kombu.connection.Connection` constructor. Any
keyword arguments given to this function will be passed through to the
:py:class:`~kombu.connection.Connection` constructor. Keyword arguments
take precedence over the configuration file.
Supported keys:
* ``credentials_secret``
* ``hostname``
* ``virtual_host``
"""
assert prefix.endswith(".")
parser = config.SpecParser({
"credentials_secret":
config.Optional(config.String),
"hostname":
config.String,
"virtual_host":
config.Optional(config.String),
})
options = parser.parse(prefix[:-1], app_config)
if options.credentials_secret:
if not secrets:
raise ValueError(
"'secrets' is required if 'credentials_secret' is set")
credentials = secrets.get_credentials(options.credentials_secret)
kwargs.setdefault("userid", credentials.username)
kwargs.setdefault("password", credentials.password)
return Connection(hostname=options.hostname,
virtual_host=options.virtual_host,
**kwargs)
def http_adapter_from_config(
app_config: config.RawConfig, prefix: str, **kwargs: Any
) -> HTTPAdapter:
"""Make an HTTPAdapter from a configuration dictionary.
The keys useful to :py:func:`http_adapter_from_config` should be prefixed,
e.g. ``http.pool_connections``, ``http.max_retries``, etc. The ``prefix``
argument specifies the prefix used. Each key is mapped to a corresponding
keyword argument on the :py:class:`~requests.adapters.HTTPAdapter`
constructor.
Supported keys:
* ``pool_connections``: The number of connections to cache (default: 10).
* ``pool_maxsize``: The maximum number of connections to keep in the pool
(default: 10).
* ``max_retries``: How many times to retry DNS lookups or connection
attempts, but never sending data (default: 0).
* ``pool_block``: Whether the connection pool will block when trying to get
a connection (default: false).
Additionally, the rules for Advocate's address filtering can be configured
with the ``filter`` sub-keys:
* ``filter.ip_allowlist``: A comma-delimited list of IP addresses (1.2.3.4)
or CIDR-notation (1.2.3.0/24) ranges that the client can always connect to
(default: anything not on the local network).
* ``filter.ip_denylist``: A comma-delimited list of IP addresses or
CIDR-notation ranges the client may never connect to (default: the local network).
* ``filter.port_allowlist``: A comma-delimited list of TCP port numbers
that the client can connect to (default: 80, 8080, 443, 8443, 8000).
* ``filter.port_denylist``: A comma-delimited list of TCP port numbers that
the client may never connect to (default: none).
* ``filter.hostname_denylist``: A comma-delimited list of hostnames that
the client may never connect to (default: none).
* ``filter.allow_ipv6``: Should the client be allowed to connect to IPv6
hosts? (default: false, note: IPv6 is tricky to apply filtering rules
comprehensively to).
"""
assert prefix.endswith(".")
parser = config.SpecParser(
{
"pool_connections": config.Optional(config.Integer, default=10),
"pool_maxsize": config.Optional(config.Integer, default=10),
"max_retries": config.Optional(config.Integer, default=0),
"pool_block": config.Optional(config.Boolean, default=False),
"filter": {
"ip_allowlist": config.Optional(config.TupleOf(ipaddress.ip_network)),
"ip_denylist": config.Optional(config.TupleOf(ipaddress.ip_network)),
"port_allowlist": config.Optional(config.TupleOf(int)),
"port_denylist": config.Optional(config.TupleOf(int)),
"hostname_denylist": config.Optional(config.TupleOf(config.String)),
"allow_ipv6": config.Optional(config.Boolean, default=False),
},
}
)
options = parser.parse(prefix[:-1], app_config)
if options.pool_connections is not None:
kwargs.setdefault("pool_connections", options.pool_connections)
if options.pool_maxsize is not None:
kwargs.setdefault("pool_maxsize", options.pool_maxsize)
if options.max_retries is not None:
kwargs.setdefault("max_retries", options.max_retries)
if options.pool_block is not None:
kwargs.setdefault("pool_block", options.pool_block)
kwargs.setdefault(
"validator",
AddrValidator(
ip_whitelist=options.filter.ip_allowlist,
ip_blacklist=options.filter.ip_denylist,
port_whitelist=options.filter.port_allowlist,
port_blacklist=options.filter.port_denylist,
hostname_blacklist=options.filter.hostname_denylist,
allow_ipv6=options.filter.allow_ipv6,
),
)
return ValidatingHTTPAdapter(**kwargs)
def pool_from_config(
app_config: config.RawConfig,
prefix: str = "memcache.",
serializer: Optional[Serializer] = None,
deserializer: Optional[Deserializer] = None,
) -> PooledClient:
"""Make a PooledClient from a configuration dictionary.
The keys useful to :py:func:`pool_from_config` should be prefixed, e.g.
``memcache.endpoint``, ``memcache.max_pool_size``, etc. The ``prefix``
argument specifies the prefix used to filter keys. Each key is mapped to a
corresponding keyword argument on the
:py:class:`~pymemcache.client.base.PooledClient` constructor.
Supported keys:
* ``endpoint`` (required): a string representing a host and port to connect
to memcached service, e.g. ``localhost:11211`` or ``127.0.0.1:11211``.
* ``max_pool_size``: an integer for the maximum pool size to use, by default
this is ``2147483648``.
* ``connect_timeout``: how long (as
:py:func:`~baseplate.lib.config.Timespan`) to wait for a connection to
memcached server. Defaults to the underlying socket default timeout.
* ``timeout``: how long (as :py:func:`~baseplate.lib.config.Timespan`) to
wait for calls on the socket connected to memcache. Defaults to the
underlying socket default timeout.
:param app_config: the raw application configuration
:param prefix: prefix for configuration keys
:param serializer: function to serialize values to strings suitable
for being stored in memcached. An example is
:py:func:`~baseplate.clients.memcache.lib.make_dump_and_compress_fn`.
:param deserializer: function to convert strings returned from
memcached to arbitrary objects, must be compatible with ``serializer``.
An example is :py:func:`~baseplate.clients.memcache.lib.decompress_and_load`.
:returns: :py:class:`pymemcache.client.base.PooledClient`
"""
assert prefix.endswith(".")
parser = config.SpecParser({
"endpoint":
config.Endpoint,
"max_pool_size":
config.Optional(config.Integer, default=None),
"connect_timeout":
config.Optional(config.TimespanWithLegacyFallback, default=None),
"timeout":
config.Optional(config.TimespanWithLegacyFallback, default=None),
"no_delay":
config.Optional(config.Boolean, default=True),
})
options = parser.parse(prefix[:-1], app_config)
return PooledClient(
server=options.endpoint.address,
connect_timeout=options.connect_timeout
and options.connect_timeout.total_seconds(),
timeout=options.timeout and options.timeout.total_seconds(),
serializer=serializer,
deserializer=deserializer,
no_delay=options.no_delay,
max_pool_size=options.max_pool_size,
)
def cluster_pool_from_config(
app_config: config.RawConfig,
prefix: str = "rediscluster.",
**kwargs: Any) -> rediscluster.ClusterConnectionPool:
"""Make a ClusterConnectionPool from a configuration dictionary.
The keys useful to :py:func:`cluster_pool_from_config` should be prefixed, e.g.
``rediscluster.url``, ``rediscluster.max_connections``, etc. The ``prefix`` argument
specifies the prefix used to filter keys. Each key is mapped to a
corresponding keyword argument on the :py:class:`rediscluster.ClusterConnectionPool`
constructor.
Supported keys:
* ``url`` (required): a URL like ``redis://localhost/0``.
* ``max_connections``: an integer maximum number of connections in the pool
* ``max_connections_per_node``: Boolean, whether max_connections should be applied
globally (False) or per node (True).
* ``skip_full_coverage_check``: Skips the check of cluster-require-full-coverage
config, useful for clusters without the CONFIG command (like aws)
* ``nodemanager_follow_cluster``: Tell the node manager to reuse the last set of
nodes it was operating on when intializing.
* ``read_from_replicas``: (Boolean) Whether the client should send all read queries to
replicas instead of just the primary
* ``timeout``: how long to wait for sockets to connect. e.g.
``200 milliseconds`` (:py:func:`~baseplate.lib.config.Timespan`)
* ``track_key_reads_sample_rate``: If greater than zero, which percentage of requests will
be inspected to keep track of hot key usage within Redis when reading.
Every command inspected will result in a write to a sorted set
(baseplate-hot-key-tracker-reads) for tracking.
* ``track_key_writes_sample_rate``: If greater than zero, which percentage of requests will
be inspected to keep track of hot key usage within Redis when writing.
Every command inspected will result in a write to a sorted set
(baseplate-hot-key-tracker-reads) for tracking.
"""
assert prefix.endswith(".")
parser = config.SpecParser({
"url":
config.String,
"max_connections":
config.Optional(config.Integer, default=50),
"max_connections_per_node":
config.Optional(config.Boolean, default=False),
"timeout":
config.Optional(config.Timespan, default=None),
"read_from_replicas":
config.Optional(config.Boolean, default=True),
"skip_full_coverage_check":
config.Optional(config.Boolean, default=True),
"nodemanager_follow_cluster":
config.Optional(config.Boolean, default=None),
"decode_responses":
config.Optional(config.Boolean, default=True),
"track_key_reads_sample_rate":
config.Optional(config.Float, default=0),
"track_key_writes_sample_rate":
config.Optional(config.Float, default=0),
})
options = parser.parse(prefix[:-1], app_config)
# We're explicitly setting a default here because of https://github.com/Grokzen/redis-py-cluster/issues/435
kwargs.setdefault("max_connections", options.max_connections)
kwargs.setdefault("decode_responses", options.decode_responses)
if options.nodemanager_follow_cluster is not None:
kwargs.setdefault("nodemanager_follow_cluster",
options.nodemanager_follow_cluster)
if options.skip_full_coverage_check is not None:
kwargs.setdefault("skip_full_coverage_check",
options.skip_full_coverage_check)
if options.timeout is not None:
kwargs.setdefault("timeout", options.timeout.total_seconds())
if options.read_from_replicas:
connection_pool = ClusterWithReadReplicasBlockingConnectionPool.from_url(
options.url, **kwargs)
else:
connection_pool = rediscluster.ClusterBlockingConnectionPool.from_url(
options.url, **kwargs)
connection_pool.track_key_reads_sample_rate = options.track_key_reads_sample_rate
connection_pool.track_key_writes_sample_rate = options.track_key_writes_sample_rate
connection_pool.read_from_replicas = options.read_from_replicas
connection_pool.skip_full_coverage_check = options.skip_full_coverage_check
return connection_pool
