def metrics_client_from_config(raw_config: config.RawConfig) -> Client:
"""Configure and return a metrics client.
This expects two configuration options:
``metrics.namespace``
The root key to prefix all metrics in this application with.
``metrics.endpoint``
A ``host:port`` pair, e.g. ``localhost:2014``. If an empty string, a
client that discards all metrics will be returned.
:param raw_config: The application configuration which should have
settings for the metrics client.
:return: A configured client.
"""
cfg = config.parse_config(
raw_config,
{
"metrics": {
"namespace": config.String,
"endpoint": config.Optional(config.Endpoint)
}
},
)
# pylint: disable=maybe-no-member
return make_client(cfg.metrics.namespace, cfg.metrics.endpoint)
def init_sentry_client_from_config(raw_config: config.RawConfig,
**kwargs: Any) -> None:
"""Configure the Sentry client.
This expects one configuration option and can take many optional ones:
``sentry.dsn``
The DSN provided by Sentry. If blank, the reporter will discard events.
``sentry.environment`` (optional)
The environment your application is running in.
``sentry.sample_rate`` (optional)
Percentage of errors to report. (e.g. "37%")
``sentry.ignore_errors`` (optional)
A comma-delimited list of exception names, unqualified (e.g.
ServerTimeout) or fully qualified (e.g.
baseplate.observers.timeout.ServerTimeout) to not notify sentry about.
Note: a minimal list of common exceptions is hard-coded in Baseplate,
this option only extends that list.
Example usage::
init_sentry_client_from_config(app_config)
:param raw_config: The application configuration which should have
settings for the error reporter.
"""
cfg = config.parse_config(
raw_config,
{
"sentry": {
"dsn":
config.Optional(config.String, default=None),
"environment":
config.Optional(config.String, default=None),
"sample_rate":
config.Optional(config.Percent, default=1),
"ignore_errors":
config.Optional(config.TupleOf(config.String), default=()),
}
},
)
if cfg.sentry.dsn:
kwargs.setdefault("dsn", cfg.sentry.dsn)
if cfg.sentry.environment:
kwargs.setdefault("environment", cfg.sentry.environment)
kwargs.setdefault("sample_rate", cfg.sentry.sample_rate)
ignore_errors: List[Union[type, str]] = []
ignore_errors.extend(ALWAYS_IGNORE_ERRORS)
ignore_errors.extend(cfg.sentry.ignore_errors)
kwargs.setdefault("ignore_errors", ignore_errors)
kwargs.setdefault("with_locals", False)
client = sentry_sdk.Client(**kwargs)
sentry_sdk.Hub.current.bind_client(client)
def make_server(server_config: Dict[str, str], listener: socket.socket,
app: Any) -> StreamServer:
"""Make a gevent server for WSGI apps."""
# pylint: disable=maybe-no-member
cfg = config.parse_config(
server_config,
{
"handler": config.Optional(config.String, default=None),
"max_concurrency": config.Integer,
"stop_timeout": config.Optional(config.Integer, default=0),
},
)
pool = Pool(size=cfg.max_concurrency)
log = LoggingLogAdapter(logger, level=logging.DEBUG)
kwargs: Dict[str, Any] = {}
if cfg.handler:
kwargs["handler_class"] = _load_factory(cfg.handler, default_name=None)
server = WSGIServer(
listener,
application=app,
spawn=pool,
log=log,
error_log=LoggingLogAdapter(logger, level=logging.ERROR),
**kwargs,
)
server.stop_timeout = cfg.stop_timeout
runtime_monitor.start(server_config, app, pool)
return server
def make_server(server_config: Dict[str, str], listener: socket.socket,
app: Any) -> StreamServer:
# pylint: disable=maybe-no-member
cfg = config.parse_config(
server_config,
{
"max_concurrency":
config.Optional(config.Integer),
"stop_timeout":
config.Optional(config.TimespanWithLegacyFallback,
default=datetime.timedelta(seconds=10)),
},
)
if cfg.max_concurrency is not None:
logger.warning(
"The max_concurrency setting is deprecated for Thrift servers. See https://git.io/Jeywc."
)
pool = Pool(size=cfg.max_concurrency)
server = GeventServer(processor=app, listener=listener, spawn=pool)
server.stop_timeout = cfg.stop_timeout.total_seconds()
runtime_monitor.start(server_config, app, pool)
return server
def secrets_store_from_config(app_config: config.RawConfig,
timeout: Optional[int] = None,
prefix: str = "secrets.") -> SecretsStore:
"""Configure and return a secrets store.
The keys useful to :py:func:`secrets_store_from_config` should be prefixed, e.g.
``secrets.url``, etc.
Supported keys:
``path``: the path to the secrets file generated by the secrets fetcher daemon.
:param app_config: The application configuration which should have
settings for the secrets store.
:param timeout: How long, in seconds, to block instantiation waiting
for the secrets data to become available (defaults to not blocking).
:param prefix: Specifies the prefix used to filter keys. Defaults
to "secrets."
:param backoff: retry backoff time for secrets file watcher. Defaults to
None, which is mapped to DEFAULT_FILEWATCHER_BACKOFF.
:param provider: The secrets provider, acceptable values are 'vault' and 'vault_csi'. Defaults to 'vault'
"""
parser: SecretParser
assert prefix.endswith(".")
config_prefix = prefix[:-1]
cfg = config.parse_config(
app_config,
{
config_prefix: {
"path":
config.Optional(config.String,
default="/var/local/secrets.json"),
"provider":
config.Optional(config.String, default="vault"),
"backoff":
config.Optional(config.Timespan),
}
},
)
options = getattr(cfg, config_prefix)
if options.backoff:
backoff = options.backoff.total_seconds()
else:
backoff = None
if options.provider == "vault_csi":
parser = parse_vault_csi
return DirectorySecretsStore(options.path,
parser,
timeout=timeout,
backoff=backoff)
return SecretsStore(options.path,
timeout=timeout,
backoff=backoff,
parser=parse_secrets_fetcher)
def from_config_and_client(
cls, raw_config: config.RawConfig, client: metrics.Client
) -> "MetricsBaseplateObserver":
cfg = config.parse_config(
raw_config,
{"metrics_observer": {"sample_rate": config.Optional(config.Percent, default=1.0)}},
)
return cls(client, sample_rate=cfg.metrics_observer.sample_rate)
def experiments_client_from_config(
app_config: config.RawConfig,
event_logger: EventLogger,
prefix: str = "experiments.") -> ExperimentsContextFactory:
"""Configure and return an :py:class:`ExperimentsContextFactory` object.
The keys useful to :py:func:`experiments_client_from_config` should be prefixed, e.g.
``experiments.path``, etc.
Supported keys:
``path``: the path to the experiment configuration file generated by the
experiment configuration fetcher daemon.
``timeout`` (optional): the time that we should wait for the file specified by
``path`` to exist. Defaults to `None` which is `infinite`.
:param raw_config: The application configuration which should have
settings for the experiments client.
:param event_logger: The EventLogger to be used to log bucketing events.
:param prefix: the prefix used to filter keys (defaults to "experiments.").
:param backoff: retry backoff time for experiments file watcher. Defaults to
None, which is mapped to DEFAULT_FILEWATCHER_BACKOFF.
"""
assert prefix.endswith(".")
config_prefix = prefix[:-1]
cfg = config.parse_config(
app_config,
{
config_prefix: {
"path":
config.Optional(config.String,
default="/var/local/experiments.json"),
"timeout":
config.Optional(config.Timespan),
"backoff":
config.Optional(config.Timespan),
}
},
)
options = getattr(cfg, config_prefix)
# pylint: disable=maybe-no-member
if options.timeout:
timeout = options.timeout.total_seconds()
else:
timeout = None
if options.backoff:
backoff = options.backoff.total_seconds()
else:
backoff = None
return ExperimentsContextFactory(options.path,
event_logger,
timeout=timeout,
backoff=backoff)
def configure_context(self, *args: Any, **kwargs: Any) -> None: # noqa: F811
"""Add a number of objects to each request's context object.
Configure and attach multiple clients to the
:py:class:`~baseplate.RequestContext` in one place. This takes a full
configuration spec like :py:func:`baseplate.lib.config.parse_config`
and will attach the specified structure onto the context object each
request.
For example, a configuration like::
baseplate = Baseplate(app_config)
baseplate.configure_context({
"cfg": {
"doggo_is_good": config.Boolean,
},
"cache": MemcachedClient(),
"cassandra": {
"foo": CassandraClient("foo_keyspace"),
"bar": CassandraClient("bar_keyspace"),
},
})
would build a context object that could be used like::
assert context.cfg.doggo_is_good == True
context.cache.get("example")
context.cassandra.foo.execute()
:param app_config: The raw stringy configuration dictionary.
:param context_spec: A specification of what the configuration should
look like.
"""
if len(args) == 1:
kwargs["context_spec"] = args[0]
elif len(args) == 2:
kwargs["app_config"] = args[0]
kwargs["context_spec"] = args[1]
else:
raise Exception("bad parameters to configure_context")
if "app_config" in kwargs:
if self._app_config:
raise Exception("pass app_config to the constructor or this method but not both")
warn_deprecated(
"Passing configuration to configure_context is deprecated in "
"favor of passing it to the Baseplate constructor"
)
app_config = kwargs["app_config"]
else:
app_config = self._app_config
context_spec = kwargs["context_spec"]
cfg = config.parse_config(app_config, context_spec)
self._context_config.update(cfg)
def main() -> NoReturn:
arg_parser = argparse.ArgumentParser(
description=sys.modules[__name__].__doc__)
arg_parser.add_argument("config_file",
type=argparse.FileType("r"),
help="path to a configuration file")
arg_parser.add_argument("--debug",
default=False,
action="store_true",
help="enable debug logging")
args = arg_parser.parse_args()
if args.debug:
level = logging.DEBUG
else:
level = logging.INFO
logging.basicConfig(level=level, format="%(message)s")
# quiet kazoo's verbose logs a bit
logging.getLogger("kazoo").setLevel(logging.WARNING)
parser = configparser.RawConfigParser(
interpolation=EnvironmentInterpolation())
parser.read_file(args.config_file)
watcher_config = dict(parser.items("live-data"))
cfg = config.parse_config(
watcher_config,
{
"nodes":
config.DictOf(
{
"source": config.String,
"dest": config.String,
"owner": config.Optional(config.UnixUser),
"group": config.Optional(config.UnixGroup),
"mode": config.Optional(config.Integer(base=8),
default=0o400), # type: ignore
})
},
)
# pylint: disable=maybe-no-member
nodes = cfg.nodes.values()
secrets = secrets_store_from_config(watcher_config, timeout=30)
zookeeper = zookeeper_client_from_config(secrets,
watcher_config,
read_only=True)
zookeeper.start()
try:
watch_zookeeper_nodes(zookeeper, nodes)
finally:
zookeeper.stop()
def from_config(cls, app_config: config.RawConfig) -> "TimeoutBaseplateObserver":
cfg = config.parse_config(
app_config,
{
"server_timeout": {
"default": config.TimespanOrInfinite,
"debug": config.Optional(config.Boolean, default=False),
"by_endpoint": config.DictOf(config.TimespanOrInfinite),
}
},
)
return cls(cfg.server_timeout)
def load_app_and_run_server() -> None:
"""Parse arguments, read configuration, and start the server."""
sys.path.append(os.getcwd())
shutdown_event = register_signal_handlers()
args = parse_args(sys.argv[1:])
with args.config_file:
config = read_config(args.config_file, args.server_name, args.app_name)
assert config.server
configure_logging(config, args.debug)
app = make_app(config.app)
listener = make_listener(args.bind)
server = make_server(config.server, listener, app)
cfg = parse_config(config.server, {"drain_time": OptionalConfig(Timespan)})
if einhorn.is_worker():
einhorn.ack_startup()
try:
# pylint: disable=cyclic-import
from baseplate.server.prometheus import start_prometheus_exporter
except ImportError:
logger.debug("Prometheus exporter not available. pip install prometheus-client to enable.")
else:
start_prometheus_exporter()
if args.reload:
reloader.start_reload_watcher(extra_files=[args.config_file.name])
# clean up leftovers from initialization before we get into requests
gc.collect()
logger.info("Listening on %s", listener.getsockname())
server.start()
try:
shutdown_event.wait()
SERVER_STATE.state = ServerLifecycle.SHUTTING_DOWN
if cfg.drain_time:
logger.debug("Draining inbound requests...")
time.sleep(cfg.drain_time.total_seconds())
finally:
logger.debug("Gracefully shutting down...")
server.stop()
logger.info("Exiting")
def test_simple_config(self):
result = config.parse_config(
self.config,
{
"simple": config.String,
"foo": {"bar": config.Integer},
"noo": {"bar": config.Optional(config.String, default="")},
"deep": {"so": {"deep": config.String}},
},
)
self.assertEqual(result.simple, "oink")
self.assertEqual(result.foo.bar, 33)
self.assertEqual(result.noo.bar, "")
self.assertEqual(result.deep.so.deep, "very")
def metrics_client_from_config(raw_config: config.RawConfig) -> Client:
"""Configure and return a metrics client.
This expects two configuration options:
``metrics.namespace``
The root key to prefix all metrics in this application with.
``metrics.endpoint``
A ``host:port`` pair, e.g. ``localhost:2014``. If an empty string, a
client that discards all metrics will be returned.
`metrics.log_if_unconfigured``
Whether to log metrics when there is no unconfigured endpoint.
Defaults to false.
`metrics.swallow_network_errors``
When false, network errors during sending to metrics collector will
cause an exception to be thrown. When true, those exceptions are logged
and swallowed instead.
Defaults to false.
:param raw_config: The application configuration which should have
settings for the metrics client.
:return: A configured client.
"""
cfg = config.parse_config(
raw_config,
{
"metrics": {
"namespace":
config.Optional(config.String, default=""),
"endpoint":
config.Optional(config.Endpoint),
"log_if_unconfigured":
config.Optional(config.Boolean, default=False),
"swallow_network_errors":
config.Optional(config.Boolean, default=False),
}
},
)
# pylint: disable=maybe-no-member
return make_client(
namespace=cfg.metrics.namespace,
endpoint=cfg.metrics.endpoint,
log_if_unconfigured=cfg.metrics.log_if_unconfigured,
swallow_network_errors=cfg.metrics.swallow_network_errors,
)
def from_config_and_client(
cls, raw_config: config.RawConfig, client: metrics.Client
) -> "TaggedMetricsBaseplateObserver":
cfg = config.parse_config(
raw_config,
{
"metrics": {
"allowlist": config.Optional(config.TupleOf(config.String), default=[]),
},
"metrics_observer": {"sample_rate": config.Optional(config.Percent, default=1.0)},
},
)
return cls(
client,
allowlist=set(cfg.metrics.allowlist) | {"client", "endpoint"},
sample_rate=cfg.metrics_observer.sample_rate,
)
def make_server(server_config: Dict[str, str], listener: socket.socket,
app: Any) -> StreamServer:
# pylint: disable=maybe-no-member
cfg = config.parse_config(
server_config,
{
"max_concurrency": config.Integer,
"stop_timeout": config.Optional(config.Integer, default=0),
},
)
pool = Pool(size=cfg.max_concurrency)
server = GeventServer(processor=app, listener=listener, spawn=pool)
server.stop_timeout = cfg.stop_timeout
runtime_monitor.start(server_config, app, pool)
return server
def make_server(server_config: Dict[str, str], listener: socket.socket,
app: Any) -> StreamServer:
# pylint: disable=maybe-no-member
cfg = config.parse_config(
server_config,
{
"max_concurrency":
config.Integer,
"stop_timeout":
config.Optional(config.TimespanWithLegacyFallback,
default=datetime.timedelta(seconds=10)),
},
)
pool = Pool(size=cfg.max_concurrency)
server = GeventServer(processor=app, listener=listener, spawn=pool)
server.stop_timeout = cfg.stop_timeout.total_seconds()
runtime_monitor.start(server_config, app, pool)
return server
def make_server(server_config: Dict[str, str], listener: socket.socket,
app: Any) -> StreamServer:
"""Make a gevent server for WSGI apps."""
# pylint: disable=maybe-no-member
cfg = config.parse_config(
server_config,
{
"handler":
config.Optional(config.String, default=None),
"max_concurrency":
config.Optional(config.Integer),
"stop_timeout":
config.Optional(config.TimespanWithLegacyFallback,
default=datetime.timedelta(seconds=10)),
},
)
if cfg.max_concurrency is not None:
raise Exception(
"The max_concurrency setting is not allowed for WSGI servers. See https://git.io/Jeywc."
)
pool = Pool()
log = LoggingLogAdapter(logger, level=logging.DEBUG)
kwargs: Dict[str, Any] = {}
if cfg.handler:
kwargs["handler_class"] = _load_factory(cfg.handler, default_name=None)
server = WSGIServer(
listener,
application=app,
spawn=pool,
log=log,
error_log=LoggingLogAdapter(logger, level=logging.ERROR),
**kwargs,
)
server.stop_timeout = cfg.stop_timeout.total_seconds()
runtime_monitor.start(server_config, app, pool)
return server
def configure_context(self, app_config: config.RawConfig,
context_spec: Dict[str, Any]) -> None:
"""Add a number of objects to each request's context object.
Configure and attach multiple clients to the
:py:class:`~baseplate.RequestContext` in one place. This takes a full
configuration spec like :py:func:`baseplate.lib.config.parse_config`
and will attach the specified structure onto the context object each
request.
For example, a configuration like::
baseplate = Baseplate()
baseplate.configure_context(app_config, {
"cfg": {
"doggo_is_good": config.Boolean,
},
"cache": MemcachedClient(),
"cassandra": {
"foo": CassandraClient(),
"bar": CassandraClient(),
},
})
would build a context object that could be used like::
assert context.cfg.doggo_is_good == True
context.cache.get("example")
context.cassandra.foo.execute()
:param config: The raw stringy configuration dictionary.
:param context_spec: A specification of what the configuration should
look like. This should only contain context clients and nested
dictionaries. Unrelated configuration values should not be included.
"""
cfg = config.parse_config(app_config, context_spec)
self._context_config.update(cfg)
def secrets_store_from_config(app_config: config.RawConfig,
timeout: Optional[int] = None,
prefix: str = "secrets.") -> SecretsStore:
"""Configure and return a secrets store.
The keys useful to :py:func:`secrets_store_from_config` should be prefixed, e.g.
``secrets.url``, etc.
Supported keys:
``path``: the path to the secrets file generated by the secrets fetcher daemon.
:param app_config: The application configuration which should have
settings for the secrets store.
:param timeout: How long, in seconds, to block instantiation waiting
for the secrets data to become available (defaults to not blocking).
:param prefix: Specifies the prefix used to filter keys. Defaults
to "secrets."
"""
assert prefix.endswith(".")
config_prefix = prefix[:-1]
cfg = config.parse_config(
app_config,
{
config_prefix: {
"path":
config.Optional(config.String,
default="/var/local/secrets.json")
}
},
)
options = getattr(cfg, config_prefix)
# pylint: disable=maybe-no-member
return SecretsStore(options.path, timeout=timeout)
def from_config(
cls, app_config: config.RawConfig) -> "TimeoutBaseplateObserver":
cfg = config.parse_config(
app_config,
{
"server_timeout": {
"default":
config.Optional(config.TimespanOrInfinite, default=None),
"debug":
config.Optional(config.Boolean, default=False),
"by_endpoint":
config.DictOf(config.TimespanOrInfinite),
}
},
)
if cfg.server_timeout.default is None:
warn_deprecated(
"No server_timeout.default configured. Defaulting to no timeout. "
"Set the default timeout to 'infinite' or a timespan like '2 seconds'. "
"This will become mandatory in Baseplate.py 2.0.")
cfg.server_timeout.default = config.InfiniteTimespan
return cls(cfg.server_timeout)
def test_bad_value(self):
with self.assertRaises(config.ConfigurationError):
config.parse_config(self.config, {"foo": {"baz": config.Integer}})
def publish_traces() -> None:
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("config_file",
type=argparse.FileType("r"),
help="path to a configuration file")
arg_parser.add_argument(
"--queue-name",
default="main",
help="name of trace queue / publisher config (default: main)",
)
arg_parser.add_argument("--debug",
default=False,
action="store_true",
help="enable debug logging")
arg_parser.add_argument(
"--app-name",
default="main",
metavar="NAME",
help="name of app to load from config_file (default: main)",
)
args = arg_parser.parse_args()
if args.debug:
level = logging.DEBUG
else:
level = logging.WARNING
logging.basicConfig(level=level)
config_parser = configparser.RawConfigParser(
interpolation=EnvironmentInterpolation())
config_parser.read_file(args.config_file)
publisher_raw_cfg = dict(
config_parser.items("trace-publisher:" + args.queue_name))
publisher_cfg = config.parse_config(
publisher_raw_cfg,
{
"zipkin_api_url":
config.DefaultFromEnv(config.Endpoint, "BASEPLATE_ZIPKIN_API_URL"),
"post_timeout":
config.Optional(config.Integer, POST_TIMEOUT_DEFAULT),
"max_batch_size":
config.Optional(config.Integer, MAX_BATCH_SIZE_DEFAULT),
"retry_limit":
config.Optional(config.Integer, RETRY_LIMIT_DEFAULT),
"max_queue_size":
config.Optional(config.Integer, MAX_QUEUE_SIZE),
},
)
trace_queue = MessageQueue(
"/traces-" + args.queue_name,
max_messages=publisher_cfg.max_queue_size,
max_message_size=MAX_SPAN_SIZE,
)
# pylint: disable=maybe-no-member
inner_batch = TraceBatch(max_size=publisher_cfg.max_batch_size)
batcher = TimeLimitedBatch(inner_batch, MAX_BATCH_AGE)
metrics_client = metrics_client_from_config(publisher_raw_cfg)
publisher = ZipkinPublisher(
publisher_cfg.zipkin_api_url.address,
metrics_client,
post_timeout=publisher_cfg.post_timeout,
)
while True:
message: Optional[bytes]
try:
message = trace_queue.get(timeout=0.2)
except TimedOutError:
message = None
try:
batcher.add(message)
except BatchFull:
serialized = batcher.serialize()
publisher.publish(serialized)
batcher.reset()
batcher.add(message)
def error_reporter_from_config(raw_config: config.RawConfig,
module_name: str) -> raven.Client:
"""Configure and return a error reporter.
This expects one configuration option and can take many optional ones:
``sentry.dsn``
The DSN provided by Sentry. If blank, the reporter will discard events.
``sentry.site`` (optional)
An arbitrary string to identify this client installation.
``sentry.environment`` (optional)
The environment your application is running in.
``sentry.exclude_paths`` (optional)
Comma-delimited list of module prefixes to ignore when discovering
where an error came from.
``sentry.include_paths`` (optional)
Comma-delimited list of paths to include for consideration when
drilling down to an exception.
``sentry.ignore_exceptions`` (optional)
Comma-delimited list of fully qualified names of exception classes
(potentially with * globs) to not report.
``sentry.sample_rate`` (optional)
Percentage of errors to report. (e.g. "37%")
``sentry.processors`` (optional)
Comma-delimited list of fully qualified names of processor classes
to apply to events before sending to Sentry.
Example usage::
error_reporter_from_config(app_config, __name__)
:param raw_config: The application configuration which should have
settings for the error reporter.
:param module_name: ``__name__`` of the root module of the application.
"""
cfg = config.parse_config(
raw_config,
{
"sentry": {
"dsn":
config.Optional(config.String, default=None),
"site":
config.Optional(config.String, default=None),
"environment":
config.Optional(config.String, default=None),
"include_paths":
config.Optional(config.String, default=None),
"exclude_paths":
config.Optional(config.String, default=None),
"ignore_exceptions":
config.Optional(config.TupleOf(config.String), default=[]),
"sample_rate":
config.Optional(config.Percent, default=1),
"processors":
config.Optional(
config.TupleOf(config.String),
default=["raven.processors.SanitizePasswordsProcessor"],
),
}
},
)
application_module = sys.modules[module_name]
directory = os.path.dirname(application_module.__file__)
release = None
while directory != "/":
try:
release = raven.fetch_git_sha(directory)
except raven.exceptions.InvalidGitRepository:
directory = os.path.dirname(directory)
else:
break
# pylint: disable=maybe-no-member
return raven.Client(
dsn=cfg.sentry.dsn,
site=cfg.sentry.site,
release=release,
environment=cfg.sentry.environment,
include_paths=cfg.sentry.include_paths,
exclude_paths=cfg.sentry.exclude_paths,
ignore_exceptions=cfg.sentry.ignore_exceptions,
sample_rate=cfg.sentry.sample_rate,
processors=cfg.sentry.processors,
)
def main() -> None:
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
"config_file", type=argparse.FileType("r"), help="path to a configuration file"
)
arg_parser.add_argument(
"--debug", default=False, action="store_true", help="enable debug logging"
)
arg_parser.add_argument(
"--once",
default=False,
action="store_true",
help="only run the fetcher once rather than as a daemon",
)
args = arg_parser.parse_args()
if args.debug:
level = logging.DEBUG
else:
level = logging.INFO
logging.basicConfig(format="%(asctime)s:%(levelname)s:%(message)s", level=level)
parser = configparser.RawConfigParser(interpolation=EnvironmentInterpolation())
parser.read_file(args.config_file)
fetcher_config = dict(parser.items("secret-fetcher"))
cfg = config.parse_config(
fetcher_config,
{
"vault": {
"url": config.DefaultFromEnv(config.String, "BASEPLATE_DEFAULT_VAULT_URL"),
"role": config.String,
"auth_type": config.Optional(
config.OneOf(**VaultClientFactory.auth_types()),
default=VaultClientFactory.auth_types()["aws"],
),
"mount_point": config.DefaultFromEnv(
config.String, "BASEPLATE_VAULT_MOUNT_POINT", fallback="aws-ec2"
),
},
"output": {
"path": config.Optional(config.String, default="/var/local/secrets.json"),
"owner": config.Optional(config.UnixUser, default=0),
"group": config.Optional(config.UnixGroup, default=0),
"mode": config.Optional(config.Integer(base=8), default=0o400), # type: ignore
},
"secrets": config.Optional(config.TupleOf(config.String), default=[]),
"callback": config.Optional(config.String),
},
)
# pylint: disable=maybe-no-member
client_factory = VaultClientFactory(
cfg.vault.url, cfg.vault.role, cfg.vault.auth_type, cfg.vault.mount_point
)
if args.once:
logger.info("Running secret fetcher once")
fetch_secrets(cfg, client_factory)
trigger_callback(cfg.callback, cfg.output.path)
else:
logger.info("Running secret fetcher as a daemon")
last_proc = None
while True:
soonest_expiration = fetch_secrets(cfg, client_factory)
last_proc = trigger_callback(cfg.callback, cfg.output.path, last_proc)
time_til_expiration = soonest_expiration - datetime.datetime.utcnow()
time_to_sleep = time_til_expiration - VAULT_TOKEN_PREFETCH_TIME
time.sleep(max(int(time_to_sleep.total_seconds()), 1))
def test_missing_key(self):
with self.assertRaises(config.ConfigurationError):
config.parse_config(self.config,
{"foo": {
"not_here": config.Integer
}})
def test_dot_in_key(self):
with self.assertRaises(AssertionError):
config.parse_config(self.config, {"foo.bar": {}})
def publish_events() -> None:
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("config_file",
type=argparse.FileType("r"),
help="path to a configuration file")
arg_parser.add_argument(
"--queue-name",
default="main",
help="name of event queue / publisher config (default: main)",
)
arg_parser.add_argument("--debug",
default=False,
action="store_true",
help="enable debug logging")
args = arg_parser.parse_args()
if args.debug:
level = logging.DEBUG
else:
level = logging.WARNING
logging.basicConfig(level=level)
config_parser = configparser.RawConfigParser(
interpolation=EnvironmentInterpolation())
config_parser.read_file(args.config_file)
raw_config = dict(config_parser.items("event-publisher:" +
args.queue_name))
cfg = config.parse_config(
raw_config,
{
"collector": {
"hostname": config.String,
"version": config.Optional(config.String, default="2"),
"scheme": config.Optional(config.String, default="https"),
},
"key": {
"name": config.String,
"secret": config.Base64
},
"max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE),
},
)
metrics_client = metrics_client_from_config(raw_config)
event_queue = MessageQueue(
"/events-" + args.queue_name,
max_messages=cfg.max_queue_size,
max_message_size=MAX_EVENT_SIZE,
)
# pylint: disable=maybe-no-member
serializer = SERIALIZER_BY_VERSION[cfg.collector.version]()
batcher = TimeLimitedBatch(serializer, MAX_BATCH_AGE)
publisher = BatchPublisher(metrics_client, cfg)
while True:
message: Optional[bytes]
try:
message = event_queue.get(timeout=0.2)
except TimedOutError:
message = None
try:
batcher.add(message)
continue
except BatchFull:
pass
serialized = batcher.serialize()
try:
publisher.publish(serialized)
except Exception:
logger.exception("Events publishing failed.")
batcher.reset()
batcher.add(message)
def test_spec_contains_invalid_object(self):
with self.assertRaises(AssertionError):
config.parse_config(self.config, {"tree_people": 37})
def test_subparsers(self):
result = config.parse_config(self.config,
{"foo": config.DictOf(config.String)})
self.assertEqual(result, {"foo": {"bar": "33", "baz": "a cool guy"}})
def start(server_config: Dict[str, str], application: Any, pool: Pool) -> None:
baseplate: Baseplate = getattr(application, "baseplate", None)
if not baseplate or not baseplate._metrics_client:
logger.info(
"No metrics client configured. Server metrics will not be sent.")
return
cfg = config.parse_config(
server_config,
{
"monitoring": {
"blocked_hub": config.Optional(config.Timespan, default=None),
"concurrency": config.Optional(config.Boolean, default=True),
"connection_pool": config.Optional(config.Boolean,
default=False),
"gc": {
"stats": config.Optional(config.Boolean, default=True),
"timing": config.Optional(config.Boolean, default=False),
"refcycle": config.Optional(config.String, default=None),
},
}
},
)
reporters: List[_Reporter] = []
if cfg.monitoring.concurrency:
reporters.append(_ConcurrencyReporter(pool))
if cfg.monitoring.connection_pool:
reporters.append(
_BaseplateReporter(baseplate.get_runtime_metric_reporters()))
if cfg.monitoring.blocked_hub is not None:
try:
reporters.append(
_BlockedGeventHubReporter(
cfg.monitoring.blocked_hub.total_seconds()))
except Exception as exc:
logger.info("monitoring.blocked_hub disabled: %s", exc)
if cfg.monitoring.gc.stats:
try:
reporters.append(_GCStatsReporter())
except Exception as exc:
logger.info("monitoring.gc.stats disabled: %s", exc)
if cfg.monitoring.gc.timing:
try:
reporters.append(_GCTimingReporter())
except Exception as exc:
logger.info("monitoring.gc.timing disabled: %s", exc)
if cfg.monitoring.gc.refcycle:
try:
reporters.append(_RefCycleReporter(cfg.monitoring.gc.refcycle))
except Exception as exc:
logger.info("monitoring.gc.refcycle disabled: %s", exc)
thread = threading.Thread(
name="Server Monitoring",
target=_report_runtime_metrics_periodically,
args=(application.baseplate._metrics_client, reporters),
)
thread.daemon = True
thread.start()
