def metrics_client_from_config(raw_config: config.RawConfig) -> Client: """Configure and return a metrics client. This expects two configuration options: ``metrics.namespace`` The root key to prefix all metrics in this application with. ``metrics.endpoint`` A ``host:port`` pair, e.g. ``localhost:2014``. If an empty string, a client that discards all metrics will be returned. :param raw_config: The application configuration which should have settings for the metrics client. :return: A configured client. """ cfg = config.parse_config( raw_config, { "metrics": { "namespace": config.String, "endpoint": config.Optional(config.Endpoint) } }, ) # pylint: disable=maybe-no-member return make_client(cfg.metrics.namespace, cfg.metrics.endpoint)
def init_sentry_client_from_config(raw_config: config.RawConfig, **kwargs: Any) -> None: """Configure the Sentry client. This expects one configuration option and can take many optional ones: ``sentry.dsn`` The DSN provided by Sentry. If blank, the reporter will discard events. ``sentry.environment`` (optional) The environment your application is running in. ``sentry.sample_rate`` (optional) Percentage of errors to report. (e.g. "37%") ``sentry.ignore_errors`` (optional) A comma-delimited list of exception names, unqualified (e.g. ServerTimeout) or fully qualified (e.g. baseplate.observers.timeout.ServerTimeout) to not notify sentry about. Note: a minimal list of common exceptions is hard-coded in Baseplate, this option only extends that list. Example usage:: init_sentry_client_from_config(app_config) :param raw_config: The application configuration which should have settings for the error reporter. """ cfg = config.parse_config( raw_config, { "sentry": { "dsn": config.Optional(config.String, default=None), "environment": config.Optional(config.String, default=None), "sample_rate": config.Optional(config.Percent, default=1), "ignore_errors": config.Optional(config.TupleOf(config.String), default=()), } }, ) if cfg.sentry.dsn: kwargs.setdefault("dsn", cfg.sentry.dsn) if cfg.sentry.environment: kwargs.setdefault("environment", cfg.sentry.environment) kwargs.setdefault("sample_rate", cfg.sentry.sample_rate) ignore_errors: List[Union[type, str]] = [] ignore_errors.extend(ALWAYS_IGNORE_ERRORS) ignore_errors.extend(cfg.sentry.ignore_errors) kwargs.setdefault("ignore_errors", ignore_errors) kwargs.setdefault("with_locals", False) client = sentry_sdk.Client(**kwargs) sentry_sdk.Hub.current.bind_client(client)
def make_server(server_config: Dict[str, str], listener: socket.socket, app: Any) -> StreamServer: """Make a gevent server for WSGI apps.""" # pylint: disable=maybe-no-member cfg = config.parse_config( server_config, { "handler": config.Optional(config.String, default=None), "max_concurrency": config.Integer, "stop_timeout": config.Optional(config.Integer, default=0), }, ) pool = Pool(size=cfg.max_concurrency) log = LoggingLogAdapter(logger, level=logging.DEBUG) kwargs: Dict[str, Any] = {} if cfg.handler: kwargs["handler_class"] = _load_factory(cfg.handler, default_name=None) server = WSGIServer( listener, application=app, spawn=pool, log=log, error_log=LoggingLogAdapter(logger, level=logging.ERROR), **kwargs, ) server.stop_timeout = cfg.stop_timeout runtime_monitor.start(server_config, app, pool) return server
def make_server(server_config: Dict[str, str], listener: socket.socket, app: Any) -> StreamServer: # pylint: disable=maybe-no-member cfg = config.parse_config( server_config, { "max_concurrency": config.Optional(config.Integer), "stop_timeout": config.Optional(config.TimespanWithLegacyFallback, default=datetime.timedelta(seconds=10)), }, ) if cfg.max_concurrency is not None: logger.warning( "The max_concurrency setting is deprecated for Thrift servers. See https://git.io/Jeywc." ) pool = Pool(size=cfg.max_concurrency) server = GeventServer(processor=app, listener=listener, spawn=pool) server.stop_timeout = cfg.stop_timeout.total_seconds() runtime_monitor.start(server_config, app, pool) return server
def secrets_store_from_config(app_config: config.RawConfig, timeout: Optional[int] = None, prefix: str = "secrets.") -> SecretsStore: """Configure and return a secrets store. The keys useful to :py:func:`secrets_store_from_config` should be prefixed, e.g. ``secrets.url``, etc. Supported keys: ``path``: the path to the secrets file generated by the secrets fetcher daemon. :param app_config: The application configuration which should have settings for the secrets store. :param timeout: How long, in seconds, to block instantiation waiting for the secrets data to become available (defaults to not blocking). :param prefix: Specifies the prefix used to filter keys. Defaults to "secrets." :param backoff: retry backoff time for secrets file watcher. Defaults to None, which is mapped to DEFAULT_FILEWATCHER_BACKOFF. :param provider: The secrets provider, acceptable values are 'vault' and 'vault_csi'. Defaults to 'vault' """ parser: SecretParser assert prefix.endswith(".") config_prefix = prefix[:-1] cfg = config.parse_config( app_config, { config_prefix: { "path": config.Optional(config.String, default="/var/local/secrets.json"), "provider": config.Optional(config.String, default="vault"), "backoff": config.Optional(config.Timespan), } }, ) options = getattr(cfg, config_prefix) if options.backoff: backoff = options.backoff.total_seconds() else: backoff = None if options.provider == "vault_csi": parser = parse_vault_csi return DirectorySecretsStore(options.path, parser, timeout=timeout, backoff=backoff) return SecretsStore(options.path, timeout=timeout, backoff=backoff, parser=parse_secrets_fetcher)
def from_config_and_client( cls, raw_config: config.RawConfig, client: metrics.Client ) -> "MetricsBaseplateObserver": cfg = config.parse_config( raw_config, {"metrics_observer": {"sample_rate": config.Optional(config.Percent, default=1.0)}}, ) return cls(client, sample_rate=cfg.metrics_observer.sample_rate)
def experiments_client_from_config( app_config: config.RawConfig, event_logger: EventLogger, prefix: str = "experiments.") -> ExperimentsContextFactory: """Configure and return an :py:class:`ExperimentsContextFactory` object. The keys useful to :py:func:`experiments_client_from_config` should be prefixed, e.g. ``experiments.path``, etc. Supported keys: ``path``: the path to the experiment configuration file generated by the experiment configuration fetcher daemon. ``timeout`` (optional): the time that we should wait for the file specified by ``path`` to exist. Defaults to `None` which is `infinite`. :param raw_config: The application configuration which should have settings for the experiments client. :param event_logger: The EventLogger to be used to log bucketing events. :param prefix: the prefix used to filter keys (defaults to "experiments."). :param backoff: retry backoff time for experiments file watcher. Defaults to None, which is mapped to DEFAULT_FILEWATCHER_BACKOFF. """ assert prefix.endswith(".") config_prefix = prefix[:-1] cfg = config.parse_config( app_config, { config_prefix: { "path": config.Optional(config.String, default="/var/local/experiments.json"), "timeout": config.Optional(config.Timespan), "backoff": config.Optional(config.Timespan), } }, ) options = getattr(cfg, config_prefix) # pylint: disable=maybe-no-member if options.timeout: timeout = options.timeout.total_seconds() else: timeout = None if options.backoff: backoff = options.backoff.total_seconds() else: backoff = None return ExperimentsContextFactory(options.path, event_logger, timeout=timeout, backoff=backoff)
def configure_context(self, *args: Any, **kwargs: Any) -> None: # noqa: F811 """Add a number of objects to each request's context object. Configure and attach multiple clients to the :py:class:`~baseplate.RequestContext` in one place. This takes a full configuration spec like :py:func:`baseplate.lib.config.parse_config` and will attach the specified structure onto the context object each request. For example, a configuration like:: baseplate = Baseplate(app_config) baseplate.configure_context({ "cfg": { "doggo_is_good": config.Boolean, }, "cache": MemcachedClient(), "cassandra": { "foo": CassandraClient("foo_keyspace"), "bar": CassandraClient("bar_keyspace"), }, }) would build a context object that could be used like:: assert context.cfg.doggo_is_good == True context.cache.get("example") context.cassandra.foo.execute() :param app_config: The raw stringy configuration dictionary. :param context_spec: A specification of what the configuration should look like. """ if len(args) == 1: kwargs["context_spec"] = args[0] elif len(args) == 2: kwargs["app_config"] = args[0] kwargs["context_spec"] = args[1] else: raise Exception("bad parameters to configure_context") if "app_config" in kwargs: if self._app_config: raise Exception("pass app_config to the constructor or this method but not both") warn_deprecated( "Passing configuration to configure_context is deprecated in " "favor of passing it to the Baseplate constructor" ) app_config = kwargs["app_config"] else: app_config = self._app_config context_spec = kwargs["context_spec"] cfg = config.parse_config(app_config, context_spec) self._context_config.update(cfg)
def main() -> NoReturn: arg_parser = argparse.ArgumentParser( description=sys.modules[__name__].__doc__) arg_parser.add_argument("config_file", type=argparse.FileType("r"), help="path to a configuration file") arg_parser.add_argument("--debug", default=False, action="store_true", help="enable debug logging") args = arg_parser.parse_args() if args.debug: level = logging.DEBUG else: level = logging.INFO logging.basicConfig(level=level, format="%(message)s") # quiet kazoo's verbose logs a bit logging.getLogger("kazoo").setLevel(logging.WARNING) parser = configparser.RawConfigParser( interpolation=EnvironmentInterpolation()) parser.read_file(args.config_file) watcher_config = dict(parser.items("live-data")) cfg = config.parse_config( watcher_config, { "nodes": config.DictOf( { "source": config.String, "dest": config.String, "owner": config.Optional(config.UnixUser), "group": config.Optional(config.UnixGroup), "mode": config.Optional(config.Integer(base=8), default=0o400), # type: ignore }) }, ) # pylint: disable=maybe-no-member nodes = cfg.nodes.values() secrets = secrets_store_from_config(watcher_config, timeout=30) zookeeper = zookeeper_client_from_config(secrets, watcher_config, read_only=True) zookeeper.start() try: watch_zookeeper_nodes(zookeeper, nodes) finally: zookeeper.stop()
def from_config(cls, app_config: config.RawConfig) -> "TimeoutBaseplateObserver": cfg = config.parse_config( app_config, { "server_timeout": { "default": config.TimespanOrInfinite, "debug": config.Optional(config.Boolean, default=False), "by_endpoint": config.DictOf(config.TimespanOrInfinite), } }, ) return cls(cfg.server_timeout)
def load_app_and_run_server() -> None: """Parse arguments, read configuration, and start the server.""" sys.path.append(os.getcwd()) shutdown_event = register_signal_handlers() args = parse_args(sys.argv[1:]) with args.config_file: config = read_config(args.config_file, args.server_name, args.app_name) assert config.server configure_logging(config, args.debug) app = make_app(config.app) listener = make_listener(args.bind) server = make_server(config.server, listener, app) cfg = parse_config(config.server, {"drain_time": OptionalConfig(Timespan)}) if einhorn.is_worker(): einhorn.ack_startup() try: # pylint: disable=cyclic-import from baseplate.server.prometheus import start_prometheus_exporter except ImportError: logger.debug("Prometheus exporter not available. pip install prometheus-client to enable.") else: start_prometheus_exporter() if args.reload: reloader.start_reload_watcher(extra_files=[args.config_file.name]) # clean up leftovers from initialization before we get into requests gc.collect() logger.info("Listening on %s", listener.getsockname()) server.start() try: shutdown_event.wait() SERVER_STATE.state = ServerLifecycle.SHUTTING_DOWN if cfg.drain_time: logger.debug("Draining inbound requests...") time.sleep(cfg.drain_time.total_seconds()) finally: logger.debug("Gracefully shutting down...") server.stop() logger.info("Exiting")
def test_simple_config(self): result = config.parse_config( self.config, { "simple": config.String, "foo": {"bar": config.Integer}, "noo": {"bar": config.Optional(config.String, default="")}, "deep": {"so": {"deep": config.String}}, }, ) self.assertEqual(result.simple, "oink") self.assertEqual(result.foo.bar, 33) self.assertEqual(result.noo.bar, "") self.assertEqual(result.deep.so.deep, "very")
def metrics_client_from_config(raw_config: config.RawConfig) -> Client: """Configure and return a metrics client. This expects two configuration options: ``metrics.namespace`` The root key to prefix all metrics in this application with. ``metrics.endpoint`` A ``host:port`` pair, e.g. ``localhost:2014``. If an empty string, a client that discards all metrics will be returned. `metrics.log_if_unconfigured`` Whether to log metrics when there is no unconfigured endpoint. Defaults to false. `metrics.swallow_network_errors`` When false, network errors during sending to metrics collector will cause an exception to be thrown. When true, those exceptions are logged and swallowed instead. Defaults to false. :param raw_config: The application configuration which should have settings for the metrics client. :return: A configured client. """ cfg = config.parse_config( raw_config, { "metrics": { "namespace": config.Optional(config.String, default=""), "endpoint": config.Optional(config.Endpoint), "log_if_unconfigured": config.Optional(config.Boolean, default=False), "swallow_network_errors": config.Optional(config.Boolean, default=False), } }, ) # pylint: disable=maybe-no-member return make_client( namespace=cfg.metrics.namespace, endpoint=cfg.metrics.endpoint, log_if_unconfigured=cfg.metrics.log_if_unconfigured, swallow_network_errors=cfg.metrics.swallow_network_errors, )
def from_config_and_client( cls, raw_config: config.RawConfig, client: metrics.Client ) -> "TaggedMetricsBaseplateObserver": cfg = config.parse_config( raw_config, { "metrics": { "allowlist": config.Optional(config.TupleOf(config.String), default=[]), }, "metrics_observer": {"sample_rate": config.Optional(config.Percent, default=1.0)}, }, ) return cls( client, allowlist=set(cfg.metrics.allowlist) | {"client", "endpoint"}, sample_rate=cfg.metrics_observer.sample_rate, )
def make_server(server_config: Dict[str, str], listener: socket.socket, app: Any) -> StreamServer: # pylint: disable=maybe-no-member cfg = config.parse_config( server_config, { "max_concurrency": config.Integer, "stop_timeout": config.Optional(config.Integer, default=0), }, ) pool = Pool(size=cfg.max_concurrency) server = GeventServer(processor=app, listener=listener, spawn=pool) server.stop_timeout = cfg.stop_timeout runtime_monitor.start(server_config, app, pool) return server
def make_server(server_config: Dict[str, str], listener: socket.socket, app: Any) -> StreamServer: # pylint: disable=maybe-no-member cfg = config.parse_config( server_config, { "max_concurrency": config.Integer, "stop_timeout": config.Optional(config.TimespanWithLegacyFallback, default=datetime.timedelta(seconds=10)), }, ) pool = Pool(size=cfg.max_concurrency) server = GeventServer(processor=app, listener=listener, spawn=pool) server.stop_timeout = cfg.stop_timeout.total_seconds() runtime_monitor.start(server_config, app, pool) return server
def make_server(server_config: Dict[str, str], listener: socket.socket, app: Any) -> StreamServer: """Make a gevent server for WSGI apps.""" # pylint: disable=maybe-no-member cfg = config.parse_config( server_config, { "handler": config.Optional(config.String, default=None), "max_concurrency": config.Optional(config.Integer), "stop_timeout": config.Optional(config.TimespanWithLegacyFallback, default=datetime.timedelta(seconds=10)), }, ) if cfg.max_concurrency is not None: raise Exception( "The max_concurrency setting is not allowed for WSGI servers. See https://git.io/Jeywc." ) pool = Pool() log = LoggingLogAdapter(logger, level=logging.DEBUG) kwargs: Dict[str, Any] = {} if cfg.handler: kwargs["handler_class"] = _load_factory(cfg.handler, default_name=None) server = WSGIServer( listener, application=app, spawn=pool, log=log, error_log=LoggingLogAdapter(logger, level=logging.ERROR), **kwargs, ) server.stop_timeout = cfg.stop_timeout.total_seconds() runtime_monitor.start(server_config, app, pool) return server
def configure_context(self, app_config: config.RawConfig, context_spec: Dict[str, Any]) -> None: """Add a number of objects to each request's context object. Configure and attach multiple clients to the :py:class:`~baseplate.RequestContext` in one place. This takes a full configuration spec like :py:func:`baseplate.lib.config.parse_config` and will attach the specified structure onto the context object each request. For example, a configuration like:: baseplate = Baseplate() baseplate.configure_context(app_config, { "cfg": { "doggo_is_good": config.Boolean, }, "cache": MemcachedClient(), "cassandra": { "foo": CassandraClient(), "bar": CassandraClient(), }, }) would build a context object that could be used like:: assert context.cfg.doggo_is_good == True context.cache.get("example") context.cassandra.foo.execute() :param config: The raw stringy configuration dictionary. :param context_spec: A specification of what the configuration should look like. This should only contain context clients and nested dictionaries. Unrelated configuration values should not be included. """ cfg = config.parse_config(app_config, context_spec) self._context_config.update(cfg)
def secrets_store_from_config(app_config: config.RawConfig, timeout: Optional[int] = None, prefix: str = "secrets.") -> SecretsStore: """Configure and return a secrets store. The keys useful to :py:func:`secrets_store_from_config` should be prefixed, e.g. ``secrets.url``, etc. Supported keys: ``path``: the path to the secrets file generated by the secrets fetcher daemon. :param app_config: The application configuration which should have settings for the secrets store. :param timeout: How long, in seconds, to block instantiation waiting for the secrets data to become available (defaults to not blocking). :param prefix: Specifies the prefix used to filter keys. Defaults to "secrets." """ assert prefix.endswith(".") config_prefix = prefix[:-1] cfg = config.parse_config( app_config, { config_prefix: { "path": config.Optional(config.String, default="/var/local/secrets.json") } }, ) options = getattr(cfg, config_prefix) # pylint: disable=maybe-no-member return SecretsStore(options.path, timeout=timeout)
def from_config( cls, app_config: config.RawConfig) -> "TimeoutBaseplateObserver": cfg = config.parse_config( app_config, { "server_timeout": { "default": config.Optional(config.TimespanOrInfinite, default=None), "debug": config.Optional(config.Boolean, default=False), "by_endpoint": config.DictOf(config.TimespanOrInfinite), } }, ) if cfg.server_timeout.default is None: warn_deprecated( "No server_timeout.default configured. Defaulting to no timeout. " "Set the default timeout to 'infinite' or a timespan like '2 seconds'. " "This will become mandatory in Baseplate.py 2.0.") cfg.server_timeout.default = config.InfiniteTimespan return cls(cfg.server_timeout)
def test_bad_value(self): with self.assertRaises(config.ConfigurationError): config.parse_config(self.config, {"foo": {"baz": config.Integer}})
def publish_traces() -> None: arg_parser = argparse.ArgumentParser() arg_parser.add_argument("config_file", type=argparse.FileType("r"), help="path to a configuration file") arg_parser.add_argument( "--queue-name", default="main", help="name of trace queue / publisher config (default: main)", ) arg_parser.add_argument("--debug", default=False, action="store_true", help="enable debug logging") arg_parser.add_argument( "--app-name", default="main", metavar="NAME", help="name of app to load from config_file (default: main)", ) args = arg_parser.parse_args() if args.debug: level = logging.DEBUG else: level = logging.WARNING logging.basicConfig(level=level) config_parser = configparser.RawConfigParser( interpolation=EnvironmentInterpolation()) config_parser.read_file(args.config_file) publisher_raw_cfg = dict( config_parser.items("trace-publisher:" + args.queue_name)) publisher_cfg = config.parse_config( publisher_raw_cfg, { "zipkin_api_url": config.DefaultFromEnv(config.Endpoint, "BASEPLATE_ZIPKIN_API_URL"), "post_timeout": config.Optional(config.Integer, POST_TIMEOUT_DEFAULT), "max_batch_size": config.Optional(config.Integer, MAX_BATCH_SIZE_DEFAULT), "retry_limit": config.Optional(config.Integer, RETRY_LIMIT_DEFAULT), "max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE), }, ) trace_queue = MessageQueue( "/traces-" + args.queue_name, max_messages=publisher_cfg.max_queue_size, max_message_size=MAX_SPAN_SIZE, ) # pylint: disable=maybe-no-member inner_batch = TraceBatch(max_size=publisher_cfg.max_batch_size) batcher = TimeLimitedBatch(inner_batch, MAX_BATCH_AGE) metrics_client = metrics_client_from_config(publisher_raw_cfg) publisher = ZipkinPublisher( publisher_cfg.zipkin_api_url.address, metrics_client, post_timeout=publisher_cfg.post_timeout, ) while True: message: Optional[bytes] try: message = trace_queue.get(timeout=0.2) except TimedOutError: message = None try: batcher.add(message) except BatchFull: serialized = batcher.serialize() publisher.publish(serialized) batcher.reset() batcher.add(message)
def error_reporter_from_config(raw_config: config.RawConfig, module_name: str) -> raven.Client: """Configure and return a error reporter. This expects one configuration option and can take many optional ones: ``sentry.dsn`` The DSN provided by Sentry. If blank, the reporter will discard events. ``sentry.site`` (optional) An arbitrary string to identify this client installation. ``sentry.environment`` (optional) The environment your application is running in. ``sentry.exclude_paths`` (optional) Comma-delimited list of module prefixes to ignore when discovering where an error came from. ``sentry.include_paths`` (optional) Comma-delimited list of paths to include for consideration when drilling down to an exception. ``sentry.ignore_exceptions`` (optional) Comma-delimited list of fully qualified names of exception classes (potentially with * globs) to not report. ``sentry.sample_rate`` (optional) Percentage of errors to report. (e.g. "37%") ``sentry.processors`` (optional) Comma-delimited list of fully qualified names of processor classes to apply to events before sending to Sentry. Example usage:: error_reporter_from_config(app_config, __name__) :param raw_config: The application configuration which should have settings for the error reporter. :param module_name: ``__name__`` of the root module of the application. """ cfg = config.parse_config( raw_config, { "sentry": { "dsn": config.Optional(config.String, default=None), "site": config.Optional(config.String, default=None), "environment": config.Optional(config.String, default=None), "include_paths": config.Optional(config.String, default=None), "exclude_paths": config.Optional(config.String, default=None), "ignore_exceptions": config.Optional(config.TupleOf(config.String), default=[]), "sample_rate": config.Optional(config.Percent, default=1), "processors": config.Optional( config.TupleOf(config.String), default=["raven.processors.SanitizePasswordsProcessor"], ), } }, ) application_module = sys.modules[module_name] directory = os.path.dirname(application_module.__file__) release = None while directory != "/": try: release = raven.fetch_git_sha(directory) except raven.exceptions.InvalidGitRepository: directory = os.path.dirname(directory) else: break # pylint: disable=maybe-no-member return raven.Client( dsn=cfg.sentry.dsn, site=cfg.sentry.site, release=release, environment=cfg.sentry.environment, include_paths=cfg.sentry.include_paths, exclude_paths=cfg.sentry.exclude_paths, ignore_exceptions=cfg.sentry.ignore_exceptions, sample_rate=cfg.sentry.sample_rate, processors=cfg.sentry.processors, )
def main() -> None: arg_parser = argparse.ArgumentParser() arg_parser.add_argument( "config_file", type=argparse.FileType("r"), help="path to a configuration file" ) arg_parser.add_argument( "--debug", default=False, action="store_true", help="enable debug logging" ) arg_parser.add_argument( "--once", default=False, action="store_true", help="only run the fetcher once rather than as a daemon", ) args = arg_parser.parse_args() if args.debug: level = logging.DEBUG else: level = logging.INFO logging.basicConfig(format="%(asctime)s:%(levelname)s:%(message)s", level=level) parser = configparser.RawConfigParser(interpolation=EnvironmentInterpolation()) parser.read_file(args.config_file) fetcher_config = dict(parser.items("secret-fetcher")) cfg = config.parse_config( fetcher_config, { "vault": { "url": config.DefaultFromEnv(config.String, "BASEPLATE_DEFAULT_VAULT_URL"), "role": config.String, "auth_type": config.Optional( config.OneOf(**VaultClientFactory.auth_types()), default=VaultClientFactory.auth_types()["aws"], ), "mount_point": config.DefaultFromEnv( config.String, "BASEPLATE_VAULT_MOUNT_POINT", fallback="aws-ec2" ), }, "output": { "path": config.Optional(config.String, default="/var/local/secrets.json"), "owner": config.Optional(config.UnixUser, default=0), "group": config.Optional(config.UnixGroup, default=0), "mode": config.Optional(config.Integer(base=8), default=0o400), # type: ignore }, "secrets": config.Optional(config.TupleOf(config.String), default=[]), "callback": config.Optional(config.String), }, ) # pylint: disable=maybe-no-member client_factory = VaultClientFactory( cfg.vault.url, cfg.vault.role, cfg.vault.auth_type, cfg.vault.mount_point ) if args.once: logger.info("Running secret fetcher once") fetch_secrets(cfg, client_factory) trigger_callback(cfg.callback, cfg.output.path) else: logger.info("Running secret fetcher as a daemon") last_proc = None while True: soonest_expiration = fetch_secrets(cfg, client_factory) last_proc = trigger_callback(cfg.callback, cfg.output.path, last_proc) time_til_expiration = soonest_expiration - datetime.datetime.utcnow() time_to_sleep = time_til_expiration - VAULT_TOKEN_PREFETCH_TIME time.sleep(max(int(time_to_sleep.total_seconds()), 1))
def test_missing_key(self): with self.assertRaises(config.ConfigurationError): config.parse_config(self.config, {"foo": { "not_here": config.Integer }})
def test_dot_in_key(self): with self.assertRaises(AssertionError): config.parse_config(self.config, {"foo.bar": {}})
def publish_events() -> None: arg_parser = argparse.ArgumentParser() arg_parser.add_argument("config_file", type=argparse.FileType("r"), help="path to a configuration file") arg_parser.add_argument( "--queue-name", default="main", help="name of event queue / publisher config (default: main)", ) arg_parser.add_argument("--debug", default=False, action="store_true", help="enable debug logging") args = arg_parser.parse_args() if args.debug: level = logging.DEBUG else: level = logging.WARNING logging.basicConfig(level=level) config_parser = configparser.RawConfigParser( interpolation=EnvironmentInterpolation()) config_parser.read_file(args.config_file) raw_config = dict(config_parser.items("event-publisher:" + args.queue_name)) cfg = config.parse_config( raw_config, { "collector": { "hostname": config.String, "version": config.Optional(config.String, default="2"), "scheme": config.Optional(config.String, default="https"), }, "key": { "name": config.String, "secret": config.Base64 }, "max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE), }, ) metrics_client = metrics_client_from_config(raw_config) event_queue = MessageQueue( "/events-" + args.queue_name, max_messages=cfg.max_queue_size, max_message_size=MAX_EVENT_SIZE, ) # pylint: disable=maybe-no-member serializer = SERIALIZER_BY_VERSION[cfg.collector.version]() batcher = TimeLimitedBatch(serializer, MAX_BATCH_AGE) publisher = BatchPublisher(metrics_client, cfg) while True: message: Optional[bytes] try: message = event_queue.get(timeout=0.2) except TimedOutError: message = None try: batcher.add(message) continue except BatchFull: pass serialized = batcher.serialize() try: publisher.publish(serialized) except Exception: logger.exception("Events publishing failed.") batcher.reset() batcher.add(message)
def test_spec_contains_invalid_object(self): with self.assertRaises(AssertionError): config.parse_config(self.config, {"tree_people": 37})
def test_subparsers(self): result = config.parse_config(self.config, {"foo": config.DictOf(config.String)}) self.assertEqual(result, {"foo": {"bar": "33", "baz": "a cool guy"}})
def start(server_config: Dict[str, str], application: Any, pool: Pool) -> None: baseplate: Baseplate = getattr(application, "baseplate", None) if not baseplate or not baseplate._metrics_client: logger.info( "No metrics client configured. Server metrics will not be sent.") return cfg = config.parse_config( server_config, { "monitoring": { "blocked_hub": config.Optional(config.Timespan, default=None), "concurrency": config.Optional(config.Boolean, default=True), "connection_pool": config.Optional(config.Boolean, default=False), "gc": { "stats": config.Optional(config.Boolean, default=True), "timing": config.Optional(config.Boolean, default=False), "refcycle": config.Optional(config.String, default=None), }, } }, ) reporters: List[_Reporter] = [] if cfg.monitoring.concurrency: reporters.append(_ConcurrencyReporter(pool)) if cfg.monitoring.connection_pool: reporters.append( _BaseplateReporter(baseplate.get_runtime_metric_reporters())) if cfg.monitoring.blocked_hub is not None: try: reporters.append( _BlockedGeventHubReporter( cfg.monitoring.blocked_hub.total_seconds())) except Exception as exc: logger.info("monitoring.blocked_hub disabled: %s", exc) if cfg.monitoring.gc.stats: try: reporters.append(_GCStatsReporter()) except Exception as exc: logger.info("monitoring.gc.stats disabled: %s", exc) if cfg.monitoring.gc.timing: try: reporters.append(_GCTimingReporter()) except Exception as exc: logger.info("monitoring.gc.timing disabled: %s", exc) if cfg.monitoring.gc.refcycle: try: reporters.append(_RefCycleReporter(cfg.monitoring.gc.refcycle)) except Exception as exc: logger.info("monitoring.gc.refcycle disabled: %s", exc) thread = threading.Thread( name="Server Monitoring", target=_report_runtime_metrics_periodically, args=(application.baseplate._metrics_client, reporters), ) thread.daemon = True thread.start()