def test_bad_value(self): with self.assertRaises(config.ConfigurationError): config.parse_config(self.config, { "foo": { "baz": config.Integer, }, })
def test_missing_key(self): with self.assertRaises(config.ConfigurationError): config.parse_config(self.config, { "foo": { "not_here": config.Integer, }, })
def secrets_store_from_config(app_config, timeout=None): """Configure and return a secrets store. This expects one configuration option: ``secrets.path`` The path to the secrets file generated by the secrets fetcher daemon. :param dict raw_config: The application configuration which should have settings for the secrets store. :param float timeout: How long, in seconds, to block instantiation waiting for the secrets data to become available (defaults to not blocking). :rtype: :py:class:`SecretsStore` """ cfg = config.parse_config( app_config, { "secrets": { "path": config.Optional(config.String, default="/var/local/secrets.json"), }, }) # pylint: disable=maybe-no-member return SecretsStore(cfg.secrets.path, timeout=timeout)
def main() -> NoReturn: """ Sidecar that tracks the age of the file monitored by live-data watcher and secrets fetcher. Use this with live_data_watcher and secrets_fetcher to monitor their files and ensure that these sidecars are not failing silently. """ arg_parser = argparse.ArgumentParser( description=sys.modules[__name__].__doc__) arg_parser.add_argument("config_file", type=argparse.FileType("r"), help="path to a configuration file") args = arg_parser.parse_args() parser = configparser.RawConfigParser() parser.read(args.config_file.name) watcher_config = dict(parser.items("file-age-watcher")) cfg = config.parse_config(watcher_config, {"file": config.DictOf({"path": config.String})}) metrics_client = metrics_client_from_config(watcher_config) while True: time.sleep(HEARTBEAT_INTERVAL) now = time.time() for name, file in cfg.file.items(): try: mtime = os.path.getmtime(file.path) except OSError: mtime = 0 age = now - mtime metrics_client.histogram( f"file-age-watcher.{name}.age").add_sample(age)
def make_server(server_config, listener, app): """Make a gevent server for WSGI apps.""" # pylint: disable=maybe-no-member cfg = config.parse_config( server_config, { "handler": config.Optional(config.String, default=None), "max_concurrency": config.Integer, "stop_timeout": config.Optional(config.Integer, default=0), }, ) pool = Pool(size=cfg.max_concurrency) log = LoggingLogAdapter(logger, level=logging.DEBUG) kwargs = {} if gevent.version_info[:2] >= (1, 1): # error_log is new in 1.1 kwargs["error_log"] = LoggingLogAdapter(logger, level=logging.ERROR) if cfg.handler: kwargs["handler_class"] = _load_factory(cfg.handler, default_name=None) server = WSGIServer(listener, application=app, spawn=pool, log=log, **kwargs) server.stop_timeout = cfg.stop_timeout runtime_monitor.start(server_config, app, pool) return server
def connection_from_config(app_config, prefix, **kwargs): """Make a Connection from a configuration dictionary. The keys useful to :py:func:`connection_from_config` should be prefixed, e.g. ``amqp.hostname`` etc. The ``prefix`` argument specifies the prefix used to filter keys. Each key is mapped to a corresponding keyword argument on the :py:class:`~kombu.connection.Connection` constructor. Any keyword arguments given to this function will be passed through to the :py:class:`~kombu.connection.Connection` constructor. Keyword arguments take precedence over the configuration file. Supported keys: * ``hostname`` * ``virtual_host`` """ assert prefix.endswith(".") config_prefix = prefix[:-1] cfg = config.parse_config(app_config, { config_prefix: { "hostname": config.String, "virtual_host": config.Optional(config.String), }, }) options = getattr(cfg, config_prefix) return Connection( hostname=options.hostname, virtual_host=options.virtual_host, **kwargs )
def exchange_from_config(app_config, prefix, **kwargs): """Make an Exchange from a configuration dictionary. The keys useful to :py:func:`exchange_from_config` should be prefixed, e.g. ``amqp.exchange_name`` etc. The ``prefix`` argument specifies the prefix used to filter keys. Each key is mapped to a corresponding keyword argument on the :py:class:`~kombu.Exchange` constructor. Any keyword arguments given to this function will be passed through to the :py:class:`~kombu.Exchange` constructor. Keyword arguments take precedence over the configuration file. Supported keys: * ``exchange_name`` * ``exchange_type`` """ assert prefix.endswith(".") config_prefix = prefix[:-1] cfg = config.parse_config(app_config, { config_prefix: { "exchange_name": config.Optional(config.String), "exchange_type": config.String, }, }) options = getattr(cfg, config_prefix) return Exchange( name=options.exchange_name or '', type=options.exchange_type, **kwargs )
def make_processor(app_config): # pragma: nocover cfg = config.parse_config(app_config, { "activity": { "window": config.Timespan, "fuzz_threshold": config.Integer, }, "redis": { "url": config.String, "max_connections": config.Optional(config.Integer, default=100), }, }) metrics_client = make_metrics_client(app_config) redis_pool = redis.BlockingConnectionPool.from_url( cfg.redis.url, max_connections=cfg.redis.max_connections, timeout=0.1, ) baseplate = Baseplate() baseplate.configure_logging() baseplate.configure_metrics(metrics_client) baseplate.add_to_context("redis", RedisContextFactory(redis_pool)) counter = ActivityCounter(cfg.activity.window.total_seconds()) handler = Handler( fuzz_threshold=cfg.activity.fuzz_threshold, counter=counter, ) processor = ActivityService.ContextProcessor(handler) event_handler = BaseplateProcessorEventHandler(logger, baseplate) processor.setEventHandler(event_handler) return processor
def make_wsgi_app(app_config): cfg = config.parse_config(app_config, { "activity": { "endpoint": config.Endpoint, }, }) metrics_client = make_metrics_client(app_config) pool = ThriftConnectionPool(cfg.activity.endpoint) baseplate = Baseplate() baseplate.configure_logging() baseplate.configure_metrics(metrics_client) baseplate.add_to_context( "activity", ThriftContextFactory(pool, ActivityService.Client)) configurator = Configurator(settings=app_config) baseplate_configurator = BaseplateConfigurator(baseplate) configurator.include(baseplate_configurator.includeme) controller = ActivityGateway() configurator.add_route("health", "/health", request_method="GET") configurator.add_view(controller.is_healthy, route_name="health", renderer="json") configurator.add_route("pixel", "/{context_id:[A-Za-z0-9_]{,40}}.png", request_method="GET") configurator.add_view(controller.pixel, route_name="pixel") return configurator.make_wsgi_app()
def metrics_client_from_config(raw_config: config.RawConfig) -> metrics.Client: """Configure and return a metrics client. This expects two configuration options: ``metrics.namespace`` The root key to prefix all metrics in this application with. ``metrics.endpoint`` A ``host:port`` pair, e.g. ``localhost:2014``. If an empty string, a client that discards all metrics will be returned. :param dict raw_config: The application configuration which should have settings for the metrics client. :return: A configured client. :rtype: :py:class:`baseplate.metrics.Client` """ cfg = config.parse_config( raw_config, { "metrics": { "namespace": config.String, "endpoint": config.Optional(config.Endpoint) } }, ) # pylint: disable=maybe-no-member return metrics.make_client(cfg.metrics.namespace, cfg.metrics.endpoint)
def hvac_factory_from_config(app_config, secrets_store, prefix="vault."): """Make an HVAC client factory from a configuration dictionary. The keys useful to :py:func:`hvac_factory_from_config` should be prefixed, e.g. ``vault.timeout``. The ``prefix`` argument specifies the prefix used to filter keys. Supported keys: * ``timeout``: How long to wait for calls to Vault. :param dict app_config: The raw application configuration. :param baseplate.secrets.SecretsStore secrets_store: A configured secrets store from which we can get a Vault authentication token. :param str prefix: The prefix for configuration keys. """ assert prefix.endswith(".") config_prefix = prefix[:-1] cfg = config.parse_config( app_config, { config_prefix: { "timeout": config.Optional(config.Timespan, default=datetime.timedelta(seconds=1)) } }, ) options = getattr(cfg, config_prefix) return HvacContextFactory(secrets_store, options.timeout)
def parse_config(app_config): return config.parse_config(app_config, { "ads_tracking": { "click_secret": config.Base64, "max_click_age": config.Timespan, }, })
def make_app(raw_config): cfg = config.parse_config(raw_config, CONFIG_SPEC) metrics_client = make_metrics_client(raw_config) dispatcher = MessageDispatcher(metrics=metrics_client) source = MessageSource(config=cfg.amqp, ) app = SocketServer( metrics=metrics_client, dispatcher=dispatcher, mac_secret=cfg.web.mac_secret, ping_interval=cfg.web.ping_interval, admin_auth=cfg.web.admin_auth, conn_shed_rate=cfg.web.conn_shed_rate, ) # register SIGUSR2 to trigger app quiescing, # useful if app processes are behind # a process manager like einhorn. def _handle_quiesce_signal(_, frame): app._quiesce({}, bypass_auth=True) signal.signal(signal.SIGUSR2, _handle_quiesce_signal) signal.siginterrupt(signal.SIGUSR2, False) source.message_handler = dispatcher.on_message_received app.status_publisher = source.send_message gevent.spawn(source.pump_messages) return app
def secrets_store_from_config(app_config, timeout=None, prefix="secrets."): """Configure and return a secrets store. The keys useful to :py:func:`secrets_store_from_config` should be prefixed, e.g. ``secrets.url``, etc. Supported keys: ``path``: the path to the secrets file generated by the secrets fetcher daemon. :param dict app_config: The application configuration which should have settings for the secrets store. :param float timeout: (Optional) How long, in seconds, to block instantiation waiting for the secrets data to become available (defaults to not blocking). :param str prefix: (Optional) specifies the prefix used to filter keys. Defaults to "secrets." :rtype: :py:class:`SecretsStore` """ assert prefix.endswith(".") config_prefix = prefix[:-1] cfg = config.parse_config( app_config, { config_prefix: { "path": config.Optional(config.String, default="/var/local/secrets.json") } }, ) options = getattr(cfg, config_prefix) # pylint: disable=maybe-no-member return SecretsStore(options.path, timeout=timeout)
def publish_traces(): arg_parser = argparse.ArgumentParser() arg_parser.add_argument("config_file", type=argparse.FileType("r"), help="path to a configuration file") arg_parser.add_argument("--queue-name", default="main", help="name of trace queue / publisher config (default: main)") arg_parser.add_argument("--debug", default=False, action="store_true", help="enable debug logging") arg_parser.add_argument("--app-name", default="main", metavar="NAME", help="name of app to load from config_file (default: main)") args = arg_parser.parse_args() if args.debug: level = logging.DEBUG else: level = logging.WARNING logging.basicConfig(level=level) config_parser = configparser.RawConfigParser() config_parser.readfp(args.config_file) publisher_raw_cfg = dict(config_parser.items("trace-publisher:" + args.queue_name)) publisher_cfg = config.parse_config(publisher_raw_cfg, { "zipkin_api_url": config.Endpoint, "post_timeout": config.Optional(config.Integer, POST_TIMEOUT_DEFAULT), "max_batch_size": config.Optional(config.Integer, MAX_BATCH_SIZE_DEFAULT), "retry_limit": config.Optional(config.Integer, RETRY_LIMIT_DEFAULT), }) trace_queue = MessageQueue( "/traces-" + args.queue_name, max_messages=MAX_QUEUE_SIZE, max_message_size=MAX_SPAN_SIZE, ) # pylint: disable=maybe-no-member inner_batch = TraceBatch(max_size=publisher_cfg.max_batch_size) batcher = TimeLimitedBatch(inner_batch, MAX_BATCH_AGE) metrics_client = metrics_client_from_config(publisher_raw_cfg) publisher = ZipkinPublisher( publisher_cfg.zipkin_api_url.address, metrics_client, post_timeout=publisher_cfg.post_timeout, ) while True: try: message = trace_queue.get(timeout=.2) except TimedOutError: message = None try: batcher.add(message) except BatchFull: serialized = batcher.serialize() publisher.publish(serialized) batcher.reset() batcher.add(message)
def thrift_pool_from_config(app_config, prefix, **kwargs): """Make a ThriftConnectionPool from a configuration dictionary. The keys useful to :py:func:`thrift_pool_from_config` should be prefixed, e.g. ``example_service.endpoint`` etc. The ``prefix`` argument specifies the prefix used to filter keys. Each key is mapped to a corresponding keyword argument on the :py:class:`ThriftConnectionPool` constructor. Any keyword arguments given to this function will be also be passed through to the constructor. Keyword arguments take precedence over the configuration file. Supported keys: * ``endpoint`` (required): A ``host:port`` pair, e.g. ``localhost:2014``, where the Thrift server can be found. * ``size``: The size of the connection pool. * ``max_age``: The oldest a connection can be before it's recycled and replaced with a new one. Written as a time span e.g. ``1 minute``. * ``timeout``: The maximum amount of time a connection attempt or RPC call can take before a TimeoutError is raised. * ``max_retries``: The maximum number of times the pool will attempt to open a connection. """ assert prefix.endswith(".") config_prefix = prefix[:-1] cfg = config.parse_config( app_config, { config_prefix: { "endpoint": config.Endpoint, "size": config.Optional(config.Integer, default=10), "max_age": config.Optional(config.Timespan, default=config.Timespan("1 minute")), "timeout": config.Optional(config.Timespan, default=config.Timespan("1 second")), "max_retries": config.Optional(config.Integer, default=3), } }, ) options = getattr(cfg, config_prefix) if options.size is not None: kwargs.setdefault("size", options.size) if options.max_age is not None: kwargs.setdefault("max_age", options.max_age.total_seconds()) if options.timeout is not None: kwargs.setdefault("timeout", options.timeout.total_seconds()) if options.max_retries is not None: kwargs.setdefault("max_retries", options.max_retries) return ThriftConnectionPool(endpoint=options.endpoint, **kwargs)
def test_subparsers(self): result = config.parse_config(self.config, { "foo": config.DictOf(config.String), }) self.assertEqual(result, { "foo": { "bar": "33", "baz": "a cool guy", }, })
def make_plugin(app_config, http, irc, salons): deploy_config = config.parse_config( app_config, { "organizations": config.TupleOf(config.String), "default_hours_start": parse_time, "default_hours_end": parse_time, "default_tz": pytz.timezone, "blackout_hours_start": parse_time, "blackout_hours_end": parse_time, }) monitor = DeployMonitor(deploy_config, irc, salons) # set up http api deploy_root = resource.Resource() http.root.putChild('deploy', deploy_root) deploy_root.putChild('status', DeployStatusListener(http.hmac_secret, monitor)) deploy_root.putChild('begin', DeployBeganListener(http, monitor)) deploy_root.putChild('end', DeployEndedListener(http, monitor)) deploy_root.putChild('abort', DeployAbortedListener(http, monitor)) deploy_root.putChild('error', DeployErrorListener(http, monitor)) deploy_root.putChild('progress', DeployProgressListener(http, monitor)) deploy_root.putChild('hold', DeployHoldListener(http, monitor)) deploy_root.putChild('unhold', DeployUnHoldListener(http, monitor)) deploy_root.putChild('hold_all', DeployHoldAllListener(http, monitor)) deploy_root.putChild('unhold_all', DeployUnholdAllListener(http, monitor)) deploy_root.putChild('send_announcement', DeploySendAnnouncementListener(http, monitor)) deploy_root.putChild('get_salon_names', DeployGetSalonNamesListener(http, monitor)) # register our irc commands irc.register_command(monitor.salonify) irc.register_command(monitor.desalonify) irc.register_command(monitor.repository) irc.register_command(monitor.help) irc.register_command(monitor.status) irc.register_command(monitor.status_all) irc.register_command(monitor.hold) irc.register_command(monitor.unhold) irc.register_command(monitor.hold_all) irc.register_command(monitor.unhold_all) irc.register_command(monitor.acquire) irc.register_command(monitor.release) irc.register_command(monitor.jump) irc.register_command(monitor.notready) irc.register_command(monitor.enqueue) irc.register_command(monitor.kick) irc.register_command(monitor.refresh) irc.register_command(monitor.refresh_all) irc.register_command(monitor.forget) irc.register_command(monitor.announce) irc.register_command(monitor.set_deploy_hours) irc.register_command(monitor.get_deploy_hours)
def start(server_config, application, pool): if not hasattr(application, "baseplate") or not application.baseplate._metrics_client: logging.info( "No metrics client configured. Server metrics will not be sent.") return cfg = config.parse_config( server_config, { "monitoring": { "blocked_hub": config.Optional(config.Timespan, default=None), "concurrency": config.Optional(config.Boolean, default=True), "gc": { "stats": config.Optional(config.Boolean, default=True), "timing": config.Optional(config.Boolean, default=False), }, } }, ) reporters = [] if cfg.monitoring.concurrency: reporters.append(_ConcurrencyReporter(pool)) if cfg.monitoring.blocked_hub is not None: try: reporters.append( _BlockedGeventHubReporter( cfg.monitoring.blocked_hub.total_seconds())) except Exception as exc: logging.info("monitoring.blocked_hub disabled: %s", exc) if cfg.monitoring.gc.stats: try: reporters.append(_GCStatsReporter()) except Exception as exc: logging.info("monitoring.gc.stats disabled: %s", exc) if cfg.monitoring.gc.timing: try: reporters.append(_GCTimingReporter()) except Exception as exc: logging.info("monitoring.gc.timing disabled: %s", exc) thread = threading.Thread( name="Server Monitoring", target=_report_runtime_metrics_periodically, args=(application.baseplate._metrics_client, reporters), ) thread.daemon = True thread.start()
def main(): arg_parser = argparse.ArgumentParser( description=sys.modules[__name__].__doc__) arg_parser.add_argument("config_file", type=argparse.FileType("r"), help="path to a configuration file") arg_parser.add_argument("--debug", default=False, action="store_true", help="enable debug logging") args = arg_parser.parse_args() if args.debug: level = logging.DEBUG else: level = logging.INFO logging.basicConfig(level=level, format="%(message)s") # quiet kazoo's verbose logs a bit logging.getLogger("kazoo").setLevel(logging.WARNING) parser = configparser.RawConfigParser() parser.readfp(args.config_file) # pylint: disable=deprecated-method watcher_config = dict(parser.items("live-data")) cfg = config.parse_config( watcher_config, { "nodes": config.DictOf( { "source": config.String, "dest": config.String, "owner": config.Optional(config.UnixUser), "group": config.Optional(config.UnixGroup), "mode": config.Optional(config.Integer(base=8), default=0o400), }) }, ) # pylint: disable=maybe-no-member nodes = cfg.nodes.values() secrets = secrets_store_from_config(watcher_config, timeout=30) zookeeper = zookeeper_client_from_config(secrets, watcher_config, read_only=True) zookeeper.start() try: watch_zookeeper_nodes(zookeeper, nodes) finally: zookeeper.stop()
def experiments_client_from_config(app_config, event_logger, prefix="experiments."): """Configure and return an :py:class:`ExperimentsContextFactory` object. The keys useful to :py:func:`experiments_client_from_config` should be prefixed, e.g. ``experiments.path``, etc. Supported keys: ``path``: the path to the experiment config file generated by the experiment config fetcher daemon. ``timeout`` (optional): the time that we should wait for the file specified by ``path`` to exist. Defaults to `None` which is `infinite`. :param dict raw_config: The application configuration which should have settings for the experiments client. :param baseplate.events.EventLogger event_logger: The EventLogger to be used to log bucketing events. :param str prefix: the prefix used to filter keys (defaults to "experiments."). :rtype: :py:class:`ExperimentsContextFactory` """ assert prefix.endswith(".") config_prefix = prefix[:-1] cfg = config.parse_config( app_config, { config_prefix: { "path": config.Optional(config.String, default="/var/local/experiments.json"), "timeout": config.Optional(config.Timespan), } }, ) options = getattr(cfg, config_prefix) # pylint: disable=maybe-no-member if options.timeout: timeout = options.timeout.total_seconds() else: timeout = None return ExperimentsContextFactory(options.path, event_logger, timeout=timeout)
def pool_from_config(app_config, prefix="redis.", **kwargs): """Make a ConnectionPool from a configuration dictionary. The keys useful to :py:func:`pool_from_config` should be prefixed, e.g. ``redis.url``, ``redis.max_connections``, etc. The ``prefix`` argument specifies the prefix used to filter keys. Each key is mapped to a corresponding keyword argument on the :py:class:`redis.ConnectionPool` constructor. Supported keys: * ``url`` (required): a URL like ``redis://localhost/0``. * ``max_connections``: an integer maximum number of connections in the pool * ``socket_connect_timeout``: a timespan of how long to wait for sockets to connect. e.g. ``200 milliseconds``. * ``socket_timeout``: a timespan of how long to wait for socket operations, e.g. ``200 milliseconds``. """ assert prefix.endswith(".") config_prefix = prefix[:-1] cfg = config.parse_config( app_config, { config_prefix: { "url": config.String, "max_connections": config.Optional(config.Integer, default=None), "socket_connect_timeout": config.Optional(config.Timespan, default=None), "socket_timeout": config.Optional(config.Timespan, default=None), } }, ) options = getattr(cfg, config_prefix) if options.max_connections is not None: kwargs.setdefault("max_connections", options.max_connections) if options.socket_connect_timeout is not None: kwargs.setdefault("socket_connect_timeout", options.socket_connect_timeout.total_seconds()) if options.socket_timeout is not None: kwargs.setdefault("socket_timeout", options.socket_timeout.total_seconds()) return redis.BlockingConnectionPool.from_url(options.url, **kwargs)
def make_server(server_config, listener, app): # pylint: disable=maybe-no-member cfg = config.parse_config( server_config, { "max_concurrency": config.Integer, "stop_timeout": config.Optional(config.Integer, default=0), }) pool = Pool(size=cfg.max_concurrency) server = GeventServer( processor=app, listener=listener, spawn=pool, ) server.stop_timeout = cfg.stop_timeout return server
def make_processor(app_config): cfg = config.parse_config(app_config, { "city_db_path": config.String, }) metrics_client = make_metrics_client(app_config) agent = diagnostics.DiagnosticsAgent() agent.register(diagnostics.LoggingDiagnosticsObserver()) agent.register(diagnostics.MetricsDiagnosticsObserver(metrics_client)) handler = Handler(cfg.city_db_path) processor = GeoipService.ContextProcessor(handler) event_handler = BaseplateProcessorEventHandler(logger, agent) processor.setEventHandler(event_handler) return processor
def make_processor(app_config): cfg = config.parse_config(app_config, { "real_random": config.Boolean, }) metrics_client = make_metrics_client(app_config) agent = diagnostics.DiagnosticsAgent() agent.register(diagnostics.LoggingDiagnosticsObserver()) agent.register(diagnostics.MetricsDiagnosticsObserver(metrics_client)) handler = Handler(real_random=cfg.real_random) processor = ToyService.ContextProcessor(handler) event_handler = BaseplateProcessorEventHandler(logger, agent) processor.setEventHandler(event_handler) return processor
def make_plugin(application, app_config): http_config = config.parse_config(app_config, { "endpoint": config.String, "hmac_secret": config.String, }) root = resource.Resource() harold = resource.Resource() root.putChild('harold', harold) site = server.Site(root) site.noisy = False site.displayTracebacks = False endpoint = serverFromString(reactor, http_config.endpoint) service = internet.StreamServerEndpointService(endpoint, site) service.setServiceParent(application) return HttpPlugin(harold, http_config.hmac_secret)
def engine_from_config(app_config, secrets=None, prefix="database."): """Make an :py:class:`~sqlalchemy.engine.Engine` from a configuration dictionary. The keys useful to :py:func:`engine_from_config` should be prefixed, e.g. ``database.url``, etc. The ``prefix`` argument specifies the prefix used to filter keys. Supported keys: * ``url``: the connection URL to the database, passed to :py:func:`~sqlalchemy.engine.url.make_url` to create the :py:class:`~sqlalchemy.engine.url.URL` used to connect to the database. * ``credentials_secret`` (optional): the key used to retrieve the database credentials from ``secrets`` as a :py:class:`~baseplate.secrets.CredentialSecret`. If this is supplied, any credentials given in ``url`` we be replaced by these. """ assert prefix.endswith(".") config_prefix = prefix[:-1] cfg = config.parse_config( app_config, { config_prefix: { "url": config.String, "credentials_secret": config.Optional(config.String), } }, ) options = getattr(cfg, config_prefix) url = make_url(options.url) if options.credentials_secret: if not secrets: raise TypeError( "'secrets' is a required argument to 'engine_from_config' " "if 'credentials_secret' is set" ) credentials = secrets.get_credentials(options.credentials_secret) url.username = credentials.username url.password = credentials.password return create_engine(url)
def make_plugin(application, app_config): slack_config = config.parse_config(app_config, { "token": config.String, }) api_client = SlackWebClient(slack_config.token) endpoint = SlackEndpoint(api_client) plugin = SlackPlugin(api_client) factory = SlackClientFactory( plugin=plugin, useragent="Harold ([email protected])", ) factory.setProtocolOptions( autoPingInterval=5, autoPingTimeout=10, ) service = ClientService(endpoint, factory) service.setServiceParent(application) return plugin
def test_simple_config(self): result = config.parse_config( self.config, { "simple": config.String, "foo": { "bar": config.Integer, }, "noo": { "bar": config.Optional(config.String, default=""), }, "deep": { "so": { "deep": config.String, }, }, }) self.assertEqual(result.simple, "oink") self.assertEqual(result.foo.bar, 33) self.assertEqual(result.noo.bar, "") self.assertEqual(result.deep.so.deep, "very")
def make_wsgi_app(app_config): cfg = config.parse_config(app_config, { "session": { "secret": config.Base64, } }) # configure pyramid configurator = Configurator(settings=app_config) configurator.include("pyramid_jinja2") configurator.set_default_csrf_options(require_csrf=True) configurator.set_session_factory(SignedCookieSessionFactory(cfg.session.secret)) authn_policy = RemoteUserAuthenticationPolicy(environ_key="HTTP_AUTHENTICATED_USER") authz_policy = ACLAuthorizationPolicy() configurator.set_authentication_policy(authn_policy) configurator.set_authorization_policy(authz_policy) configurator.add_request_method(get_authenticated_user, "user", reify=True) configurator.add_static_view(name="static", path="condor:static/") configurator.add_route("home", "/") configurator.add_route("polls", "/polls") configurator.add_route("poll", "/polls/{id:\d+}") configurator.scan("condor.views") # configure baseplate metrics_client = make_metrics_client(app_config) baseplate = Baseplate() baseplate.configure_logging() baseplate.configure_metrics(metrics_client) engine = engine_from_config(app_config, prefix="database.") baseplate.add_to_context("db", SQLAlchemySessionContextFactory(engine)) baseplate_configurator = BaseplateConfigurator(baseplate) configurator.include(baseplate_configurator.includeme) return configurator.make_wsgi_app()
def make_processor(app_config): # pragma: nocover cfg = config.parse_config(app_config, { "activity": { "window": config.Timespan, }, "tracing": { "endpoint": config.Optional(config.Endpoint), "service_name": config.String, }, "redis": { "url": config.String, "max_connections": config.Optional(config.Integer, default=100), }, }) metrics_client = metrics_client_from_config(app_config) tracing_client = tracing_client_from_config(app_config) error_reporter = error_reporter_from_config(app_config, __name__) redis_pool = redis.BlockingConnectionPool.from_url( cfg.redis.url, max_connections=cfg.redis.max_connections, timeout=0.1, ) baseplate = Baseplate() baseplate.configure_logging() baseplate.configure_metrics(metrics_client) baseplate.configure_tracing(tracing_client) baseplate.configure_error_reporting(error_reporter) baseplate.add_to_context("redis", RedisContextFactory(redis_pool)) counter = ActivityCounter(cfg.activity.window.total_seconds()) handler = Handler(counter=counter) processor = ActivityService.ContextProcessor(handler) event_handler = BaseplateProcessorEventHandler(logger, baseplate) processor.setEventHandler(event_handler) return processor
def make_processor(app_config): cfg = config.parse_config(app_config, { 'redis_endpoint': config.String, 'fuzz_threshold': config.Integer, 'activity_window': config.Integer, }) metrics_client = make_metrics_client(app_config) agent = diagnostics.DiagnosticsAgent() agent.register(diagnostics.LoggingDiagnosticsObserver()) agent.register(diagnostics.MetricsDiagnosticsObserver(metrics_client)) handler = Handler( redis_endpoint=cfg.redis_endpoint, fuzz_threshold=cfg.fuzz_threshold, activity_window=cfg.activity_window, ) processor = ActivityService.ContextProcessor(handler) event_handler = BaseplateProcessorEventHandler(logger, agent) processor.setEventHandler(event_handler) return processor
def test_simple_config(self): result = config.parse_config(self.config, { "simple": config.String, "foo": { "bar": config.Integer, }, "noo": { "bar": config.String, }, "deep": { "so": { "deep": config.String, }, }, }) self.assertEqual(result.simple, "oink") self.assertEqual(result.foo.bar, 33) self.assertEqual(result.noo.bar, "") self.assertEqual(result.deep.so.deep, "very")
def configure_context(self, app_config: config.RawConfig, context_spec: Dict) -> None: """Add a number of objects to each request's context object. Configure and attach multiple clients to the :term:`context object` in one place. This takes a full configuration spec like :py:func:`baseplate.config.parse_config` and will attach the specified structure onto the context object each request. For example, a configuration like:: baseplate = Baseplate() baseplate.configure_context(app_config, { "cfg": { "doggo_is_good": config.Boolean, }, "cache": MemcachedClient(), "cassandra": { "foo": CassandraClient(), "bar": CassandraClient(), }, }) would build a context object that could be used like:: assert context.cfg.doggo_is_good == True context.cache.get("example") context.cassandra.foo.execute() :param config: The raw stringy configuration dictionary. :param context_spec: A specification of what the config should look like. This should only contain context clients and nested dictionaries. Unrelated configuration values should not be included. """ cfg = config.parse_config(app_config, context_spec) self._context_config.update(cfg)
def make_app(raw_config): cfg = config.parse_config(raw_config, CONFIG_SPEC) metrics_client = make_metrics_client(raw_config) dispatcher = MessageDispatcher(metrics=metrics_client) source = MessageSource( config=cfg.amqp, ) app = SocketServer( metrics=metrics_client, dispatcher=dispatcher, mac_secret=cfg.web.mac_secret, ping_interval=cfg.web.ping_interval, ) source.message_handler = dispatcher.on_message_received app.status_publisher = source.send_message gevent.spawn(source.pump_messages) return app
def cluster_from_config(app_config, prefix="cassandra.", **kwargs): """Make a Cluster from a configuration dictionary. The keys useful to :py:func:`cluster_from_config` should be prefixed, e.g. ``cassandra.contact_points`` etc. The ``prefix`` argument specifies the prefix used to filter keys. Each key is mapped to a corresponding keyword argument on the :py:class:`~cassandra.cluster.Cluster` constructor. Any keyword arguments given to this function will be passed through to the :py:class:`~cassandra.cluster.Cluster` constructor. Keyword arguments take precedence over the configuration file. Supported keys: * ``contact_points`` (required): comma delimited list of contact points to try connecting for cluster discovery * ``port``: The server-side port to open connections to. """ assert prefix.endswith(".") config_prefix = prefix[:-1] cfg = config.parse_config( app_config, { config_prefix: { "contact_points": config.TupleOf(config.String), "port": config.Optional(config.Integer, default=None), } }, ) options = getattr(cfg, config_prefix) if options.port: kwargs.setdefault("port", options.port) return Cluster(options.contact_points, **kwargs)
def test_spec_contains_invalid_object(self): with self.assertRaises(AssertionError): config.parse_config(self.config, { "tree_people": 37, })
def test_dot_in_key(self): with self.assertRaises(AssertionError): config.parse_config(self.config, { "foo.bar": {}, })