def test_bad_value(self):
with self.assertRaises(config.ConfigurationError):
config.parse_config(self.config, {
"foo": {
"baz": config.Integer,
},
})
def test_missing_key(self):
with self.assertRaises(config.ConfigurationError):
config.parse_config(self.config, {
"foo": {
"not_here": config.Integer,
},
})
def test_bad_value(self):
with self.assertRaises(config.ConfigurationError):
config.parse_config(self.config, {
"foo": {
"baz": config.Integer,
},
})
def test_missing_key(self):
with self.assertRaises(config.ConfigurationError):
config.parse_config(self.config, {
"foo": {
"not_here": config.Integer,
},
})
def secrets_store_from_config(app_config, timeout=None):
"""Configure and return a secrets store.
This expects one configuration option:
``secrets.path``
The path to the secrets file generated by the secrets fetcher daemon.
:param dict raw_config: The application configuration which should have
settings for the secrets store.
:param float timeout: How long, in seconds, to block instantiation waiting
for the secrets data to become available (defaults to not blocking).
:rtype: :py:class:`SecretsStore`
"""
cfg = config.parse_config(
app_config, {
"secrets": {
"path":
config.Optional(config.String,
default="/var/local/secrets.json"),
},
})
# pylint: disable=maybe-no-member
return SecretsStore(cfg.secrets.path, timeout=timeout)
def main() -> NoReturn:
"""
Sidecar that tracks the age of the file monitored by live-data watcher and secrets fetcher.
Use this with live_data_watcher and secrets_fetcher to monitor their files and ensure that
these sidecars are not failing silently.
"""
arg_parser = argparse.ArgumentParser(
description=sys.modules[__name__].__doc__)
arg_parser.add_argument("config_file",
type=argparse.FileType("r"),
help="path to a configuration file")
args = arg_parser.parse_args()
parser = configparser.RawConfigParser()
parser.read(args.config_file.name)
watcher_config = dict(parser.items("file-age-watcher"))
cfg = config.parse_config(watcher_config,
{"file": config.DictOf({"path": config.String})})
metrics_client = metrics_client_from_config(watcher_config)
while True:
time.sleep(HEARTBEAT_INTERVAL)
now = time.time()
for name, file in cfg.file.items():
try:
mtime = os.path.getmtime(file.path)
except OSError:
mtime = 0
age = now - mtime
metrics_client.histogram(
f"file-age-watcher.{name}.age").add_sample(age)
def make_server(server_config, listener, app):
"""Make a gevent server for WSGI apps."""
# pylint: disable=maybe-no-member
cfg = config.parse_config(
server_config,
{
"handler": config.Optional(config.String, default=None),
"max_concurrency": config.Integer,
"stop_timeout": config.Optional(config.Integer, default=0),
},
)
pool = Pool(size=cfg.max_concurrency)
log = LoggingLogAdapter(logger, level=logging.DEBUG)
kwargs = {}
if gevent.version_info[:2] >= (1, 1): # error_log is new in 1.1
kwargs["error_log"] = LoggingLogAdapter(logger, level=logging.ERROR)
if cfg.handler:
kwargs["handler_class"] = _load_factory(cfg.handler, default_name=None)
server = WSGIServer(listener, application=app, spawn=pool, log=log, **kwargs)
server.stop_timeout = cfg.stop_timeout
runtime_monitor.start(server_config, app, pool)
return server
def connection_from_config(app_config, prefix, **kwargs):
"""Make a Connection from a configuration dictionary.
The keys useful to :py:func:`connection_from_config` should be prefixed,
e.g. ``amqp.hostname`` etc. The ``prefix`` argument specifies the
prefix used to filter keys. Each key is mapped to a corresponding keyword
argument on the :py:class:`~kombu.connection.Connection` constructor. Any
keyword arguments given to this function will be passed through to the
:py:class:`~kombu.connection.Connection` constructor. Keyword arguments
take precedence over the configuration file.
Supported keys:
* ``hostname``
* ``virtual_host``
"""
assert prefix.endswith(".")
config_prefix = prefix[:-1]
cfg = config.parse_config(app_config, {
config_prefix: {
"hostname": config.String,
"virtual_host": config.Optional(config.String),
},
})
options = getattr(cfg, config_prefix)
return Connection(
hostname=options.hostname,
virtual_host=options.virtual_host,
**kwargs
)
def exchange_from_config(app_config, prefix, **kwargs):
"""Make an Exchange from a configuration dictionary.
The keys useful to :py:func:`exchange_from_config` should be prefixed,
e.g. ``amqp.exchange_name`` etc. The ``prefix`` argument specifies the
prefix used to filter keys. Each key is mapped to a corresponding keyword
argument on the :py:class:`~kombu.Exchange` constructor. Any keyword
arguments given to this function will be passed through to the
:py:class:`~kombu.Exchange` constructor. Keyword arguments take precedence
over the configuration file.
Supported keys:
* ``exchange_name``
* ``exchange_type``
"""
assert prefix.endswith(".")
config_prefix = prefix[:-1]
cfg = config.parse_config(app_config, {
config_prefix: {
"exchange_name": config.Optional(config.String),
"exchange_type": config.String,
},
})
options = getattr(cfg, config_prefix)
return Exchange(
name=options.exchange_name or '',
type=options.exchange_type,
**kwargs
)
def make_processor(app_config): # pragma: nocover
cfg = config.parse_config(app_config, {
"activity": {
"window": config.Timespan,
"fuzz_threshold": config.Integer,
},
"redis": {
"url": config.String,
"max_connections": config.Optional(config.Integer, default=100),
},
})
metrics_client = make_metrics_client(app_config)
redis_pool = redis.BlockingConnectionPool.from_url(
cfg.redis.url,
max_connections=cfg.redis.max_connections,
timeout=0.1,
)
baseplate = Baseplate()
baseplate.configure_logging()
baseplate.configure_metrics(metrics_client)
baseplate.add_to_context("redis", RedisContextFactory(redis_pool))
counter = ActivityCounter(cfg.activity.window.total_seconds())
handler = Handler(
fuzz_threshold=cfg.activity.fuzz_threshold,
counter=counter,
)
processor = ActivityService.ContextProcessor(handler)
event_handler = BaseplateProcessorEventHandler(logger, baseplate)
processor.setEventHandler(event_handler)
return processor
def make_wsgi_app(app_config):
cfg = config.parse_config(app_config, {
"activity": {
"endpoint": config.Endpoint,
},
})
metrics_client = make_metrics_client(app_config)
pool = ThriftConnectionPool(cfg.activity.endpoint)
baseplate = Baseplate()
baseplate.configure_logging()
baseplate.configure_metrics(metrics_client)
baseplate.add_to_context(
"activity", ThriftContextFactory(pool, ActivityService.Client))
configurator = Configurator(settings=app_config)
baseplate_configurator = BaseplateConfigurator(baseplate)
configurator.include(baseplate_configurator.includeme)
controller = ActivityGateway()
configurator.add_route("health", "/health", request_method="GET")
configurator.add_view(controller.is_healthy,
route_name="health",
renderer="json")
configurator.add_route("pixel",
"/{context_id:[A-Za-z0-9_]{,40}}.png",
request_method="GET")
configurator.add_view(controller.pixel, route_name="pixel")
return configurator.make_wsgi_app()
def metrics_client_from_config(raw_config: config.RawConfig) -> metrics.Client:
"""Configure and return a metrics client.
This expects two configuration options:
``metrics.namespace``
The root key to prefix all metrics in this application with.
``metrics.endpoint``
A ``host:port`` pair, e.g. ``localhost:2014``. If an empty string, a
client that discards all metrics will be returned.
:param dict raw_config: The application configuration which should have
settings for the metrics client.
:return: A configured client.
:rtype: :py:class:`baseplate.metrics.Client`
"""
cfg = config.parse_config(
raw_config,
{
"metrics": {
"namespace": config.String,
"endpoint": config.Optional(config.Endpoint)
}
},
)
# pylint: disable=maybe-no-member
return metrics.make_client(cfg.metrics.namespace, cfg.metrics.endpoint)
def hvac_factory_from_config(app_config, secrets_store, prefix="vault."):
"""Make an HVAC client factory from a configuration dictionary.
The keys useful to :py:func:`hvac_factory_from_config` should be prefixed,
e.g. ``vault.timeout``. The ``prefix`` argument specifies the prefix used
to filter keys.
Supported keys:
* ``timeout``: How long to wait for calls to Vault.
:param dict app_config: The raw application configuration.
:param baseplate.secrets.SecretsStore secrets_store: A configured secrets
store from which we can get a Vault authentication token.
:param str prefix: The prefix for configuration keys.
"""
assert prefix.endswith(".")
config_prefix = prefix[:-1]
cfg = config.parse_config(
app_config,
{
config_prefix: {
"timeout":
config.Optional(config.Timespan,
default=datetime.timedelta(seconds=1))
}
},
)
options = getattr(cfg, config_prefix)
return HvacContextFactory(secrets_store, options.timeout)
def parse_config(app_config):
return config.parse_config(app_config, {
"ads_tracking": {
"click_secret": config.Base64,
"max_click_age": config.Timespan,
},
})
def make_app(raw_config):
cfg = config.parse_config(raw_config, CONFIG_SPEC)
metrics_client = make_metrics_client(raw_config)
dispatcher = MessageDispatcher(metrics=metrics_client)
source = MessageSource(config=cfg.amqp, )
app = SocketServer(
metrics=metrics_client,
dispatcher=dispatcher,
mac_secret=cfg.web.mac_secret,
ping_interval=cfg.web.ping_interval,
admin_auth=cfg.web.admin_auth,
conn_shed_rate=cfg.web.conn_shed_rate,
)
# register SIGUSR2 to trigger app quiescing,
# useful if app processes are behind
# a process manager like einhorn.
def _handle_quiesce_signal(_, frame):
app._quiesce({}, bypass_auth=True)
signal.signal(signal.SIGUSR2, _handle_quiesce_signal)
signal.siginterrupt(signal.SIGUSR2, False)
source.message_handler = dispatcher.on_message_received
app.status_publisher = source.send_message
gevent.spawn(source.pump_messages)
return app
def secrets_store_from_config(app_config, timeout=None, prefix="secrets."):
"""Configure and return a secrets store.
The keys useful to :py:func:`secrets_store_from_config` should be prefixed, e.g.
``secrets.url``, etc.
Supported keys:
``path``: the path to the secrets file generated by the secrets fetcher daemon.
:param dict app_config: The application configuration which should have
settings for the secrets store.
:param float timeout: (Optional) How long, in seconds, to block instantiation waiting
for the secrets data to become available (defaults to not blocking).
:param str prefix: (Optional) specifies the prefix used to filter keys. Defaults
to "secrets."
:rtype: :py:class:`SecretsStore`
"""
assert prefix.endswith(".")
config_prefix = prefix[:-1]
cfg = config.parse_config(
app_config,
{
config_prefix: {
"path": config.Optional(config.String, default="/var/local/secrets.json")
}
},
)
options = getattr(cfg, config_prefix)
# pylint: disable=maybe-no-member
return SecretsStore(options.path, timeout=timeout)
def publish_traces():
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("config_file", type=argparse.FileType("r"),
help="path to a configuration file")
arg_parser.add_argument("--queue-name", default="main",
help="name of trace queue / publisher config (default: main)")
arg_parser.add_argument("--debug", default=False, action="store_true",
help="enable debug logging")
arg_parser.add_argument("--app-name", default="main", metavar="NAME",
help="name of app to load from config_file (default: main)")
args = arg_parser.parse_args()
if args.debug:
level = logging.DEBUG
else:
level = logging.WARNING
logging.basicConfig(level=level)
config_parser = configparser.RawConfigParser()
config_parser.readfp(args.config_file)
publisher_raw_cfg = dict(config_parser.items("trace-publisher:" + args.queue_name))
publisher_cfg = config.parse_config(publisher_raw_cfg, {
"zipkin_api_url": config.Endpoint,
"post_timeout": config.Optional(config.Integer, POST_TIMEOUT_DEFAULT),
"max_batch_size": config.Optional(config.Integer, MAX_BATCH_SIZE_DEFAULT),
"retry_limit": config.Optional(config.Integer, RETRY_LIMIT_DEFAULT),
})
trace_queue = MessageQueue(
"/traces-" + args.queue_name,
max_messages=MAX_QUEUE_SIZE,
max_message_size=MAX_SPAN_SIZE,
)
# pylint: disable=maybe-no-member
inner_batch = TraceBatch(max_size=publisher_cfg.max_batch_size)
batcher = TimeLimitedBatch(inner_batch, MAX_BATCH_AGE)
metrics_client = metrics_client_from_config(publisher_raw_cfg)
publisher = ZipkinPublisher(
publisher_cfg.zipkin_api_url.address,
metrics_client,
post_timeout=publisher_cfg.post_timeout,
)
while True:
try:
message = trace_queue.get(timeout=.2)
except TimedOutError:
message = None
try:
batcher.add(message)
except BatchFull:
serialized = batcher.serialize()
publisher.publish(serialized)
batcher.reset()
batcher.add(message)
def thrift_pool_from_config(app_config, prefix, **kwargs):
"""Make a ThriftConnectionPool from a configuration dictionary.
The keys useful to :py:func:`thrift_pool_from_config` should be prefixed,
e.g. ``example_service.endpoint`` etc. The ``prefix`` argument specifies
the prefix used to filter keys. Each key is mapped to a corresponding
keyword argument on the :py:class:`ThriftConnectionPool` constructor. Any
keyword arguments given to this function will be also be passed through to
the constructor. Keyword arguments take precedence over the configuration
file.
Supported keys:
* ``endpoint`` (required): A ``host:port`` pair, e.g. ``localhost:2014``,
where the Thrift server can be found.
* ``size``: The size of the connection pool.
* ``max_age``: The oldest a connection can be before it's recycled and
replaced with a new one. Written as a time span e.g. ``1 minute``.
* ``timeout``: The maximum amount of time a connection attempt or RPC call
can take before a TimeoutError is raised.
* ``max_retries``: The maximum number of times the pool will attempt to
open a connection.
"""
assert prefix.endswith(".")
config_prefix = prefix[:-1]
cfg = config.parse_config(
app_config,
{
config_prefix: {
"endpoint":
config.Endpoint,
"size":
config.Optional(config.Integer, default=10),
"max_age":
config.Optional(config.Timespan,
default=config.Timespan("1 minute")),
"timeout":
config.Optional(config.Timespan,
default=config.Timespan("1 second")),
"max_retries":
config.Optional(config.Integer, default=3),
}
},
)
options = getattr(cfg, config_prefix)
if options.size is not None:
kwargs.setdefault("size", options.size)
if options.max_age is not None:
kwargs.setdefault("max_age", options.max_age.total_seconds())
if options.timeout is not None:
kwargs.setdefault("timeout", options.timeout.total_seconds())
if options.max_retries is not None:
kwargs.setdefault("max_retries", options.max_retries)
return ThriftConnectionPool(endpoint=options.endpoint, **kwargs)
def test_subparsers(self):
result = config.parse_config(self.config, {
"foo": config.DictOf(config.String),
})
self.assertEqual(result, {
"foo": {
"bar": "33",
"baz": "a cool guy",
},
})
def test_subparsers(self):
result = config.parse_config(self.config, {
"foo": config.DictOf(config.String),
})
self.assertEqual(result, {
"foo": {
"bar": "33",
"baz": "a cool guy",
},
})
def make_plugin(app_config, http, irc, salons):
deploy_config = config.parse_config(
app_config, {
"organizations": config.TupleOf(config.String),
"default_hours_start": parse_time,
"default_hours_end": parse_time,
"default_tz": pytz.timezone,
"blackout_hours_start": parse_time,
"blackout_hours_end": parse_time,
})
monitor = DeployMonitor(deploy_config, irc, salons)
# set up http api
deploy_root = resource.Resource()
http.root.putChild('deploy', deploy_root)
deploy_root.putChild('status',
DeployStatusListener(http.hmac_secret, monitor))
deploy_root.putChild('begin', DeployBeganListener(http, monitor))
deploy_root.putChild('end', DeployEndedListener(http, monitor))
deploy_root.putChild('abort', DeployAbortedListener(http, monitor))
deploy_root.putChild('error', DeployErrorListener(http, monitor))
deploy_root.putChild('progress', DeployProgressListener(http, monitor))
deploy_root.putChild('hold', DeployHoldListener(http, monitor))
deploy_root.putChild('unhold', DeployUnHoldListener(http, monitor))
deploy_root.putChild('hold_all', DeployHoldAllListener(http, monitor))
deploy_root.putChild('unhold_all', DeployUnholdAllListener(http, monitor))
deploy_root.putChild('send_announcement',
DeploySendAnnouncementListener(http, monitor))
deploy_root.putChild('get_salon_names',
DeployGetSalonNamesListener(http, monitor))
# register our irc commands
irc.register_command(monitor.salonify)
irc.register_command(monitor.desalonify)
irc.register_command(monitor.repository)
irc.register_command(monitor.help)
irc.register_command(monitor.status)
irc.register_command(monitor.status_all)
irc.register_command(monitor.hold)
irc.register_command(monitor.unhold)
irc.register_command(monitor.hold_all)
irc.register_command(monitor.unhold_all)
irc.register_command(monitor.acquire)
irc.register_command(monitor.release)
irc.register_command(monitor.jump)
irc.register_command(monitor.notready)
irc.register_command(monitor.enqueue)
irc.register_command(monitor.kick)
irc.register_command(monitor.refresh)
irc.register_command(monitor.refresh_all)
irc.register_command(monitor.forget)
irc.register_command(monitor.announce)
irc.register_command(monitor.set_deploy_hours)
irc.register_command(monitor.get_deploy_hours)
def start(server_config, application, pool):
if not hasattr(application,
"baseplate") or not application.baseplate._metrics_client:
logging.info(
"No metrics client configured. Server metrics will not be sent.")
return
cfg = config.parse_config(
server_config,
{
"monitoring": {
"blocked_hub": config.Optional(config.Timespan, default=None),
"concurrency": config.Optional(config.Boolean, default=True),
"gc": {
"stats": config.Optional(config.Boolean, default=True),
"timing": config.Optional(config.Boolean, default=False),
},
}
},
)
reporters = []
if cfg.monitoring.concurrency:
reporters.append(_ConcurrencyReporter(pool))
if cfg.monitoring.blocked_hub is not None:
try:
reporters.append(
_BlockedGeventHubReporter(
cfg.monitoring.blocked_hub.total_seconds()))
except Exception as exc:
logging.info("monitoring.blocked_hub disabled: %s", exc)
if cfg.monitoring.gc.stats:
try:
reporters.append(_GCStatsReporter())
except Exception as exc:
logging.info("monitoring.gc.stats disabled: %s", exc)
if cfg.monitoring.gc.timing:
try:
reporters.append(_GCTimingReporter())
except Exception as exc:
logging.info("monitoring.gc.timing disabled: %s", exc)
thread = threading.Thread(
name="Server Monitoring",
target=_report_runtime_metrics_periodically,
args=(application.baseplate._metrics_client, reporters),
)
thread.daemon = True
thread.start()
def main():
arg_parser = argparse.ArgumentParser(
description=sys.modules[__name__].__doc__)
arg_parser.add_argument("config_file",
type=argparse.FileType("r"),
help="path to a configuration file")
arg_parser.add_argument("--debug",
default=False,
action="store_true",
help="enable debug logging")
args = arg_parser.parse_args()
if args.debug:
level = logging.DEBUG
else:
level = logging.INFO
logging.basicConfig(level=level, format="%(message)s")
# quiet kazoo's verbose logs a bit
logging.getLogger("kazoo").setLevel(logging.WARNING)
parser = configparser.RawConfigParser()
parser.readfp(args.config_file) # pylint: disable=deprecated-method
watcher_config = dict(parser.items("live-data"))
cfg = config.parse_config(
watcher_config,
{
"nodes":
config.DictOf(
{
"source": config.String,
"dest": config.String,
"owner": config.Optional(config.UnixUser),
"group": config.Optional(config.UnixGroup),
"mode": config.Optional(config.Integer(base=8),
default=0o400),
})
},
)
# pylint: disable=maybe-no-member
nodes = cfg.nodes.values()
secrets = secrets_store_from_config(watcher_config, timeout=30)
zookeeper = zookeeper_client_from_config(secrets,
watcher_config,
read_only=True)
zookeeper.start()
try:
watch_zookeeper_nodes(zookeeper, nodes)
finally:
zookeeper.stop()
def experiments_client_from_config(app_config,
event_logger,
prefix="experiments."):
"""Configure and return an :py:class:`ExperimentsContextFactory` object.
The keys useful to :py:func:`experiments_client_from_config` should be prefixed, e.g.
``experiments.path``, etc.
Supported keys:
``path``: the path to the experiment config file generated by the experiment
config fetcher daemon.
``timeout`` (optional): the time that we should wait for the file specified by
``path`` to exist. Defaults to `None` which is `infinite`.
:param dict raw_config: The application configuration which should have
settings for the experiments client.
:param baseplate.events.EventLogger event_logger: The EventLogger to be used
to log bucketing events.
:param str prefix: the prefix used to filter keys (defaults to "experiments.").
:rtype: :py:class:`ExperimentsContextFactory`
"""
assert prefix.endswith(".")
config_prefix = prefix[:-1]
cfg = config.parse_config(
app_config,
{
config_prefix: {
"path":
config.Optional(config.String,
default="/var/local/experiments.json"),
"timeout":
config.Optional(config.Timespan),
}
},
)
options = getattr(cfg, config_prefix)
# pylint: disable=maybe-no-member
if options.timeout:
timeout = options.timeout.total_seconds()
else:
timeout = None
return ExperimentsContextFactory(options.path,
event_logger,
timeout=timeout)
def pool_from_config(app_config, prefix="redis.", **kwargs):
"""Make a ConnectionPool from a configuration dictionary.
The keys useful to :py:func:`pool_from_config` should be prefixed, e.g.
``redis.url``, ``redis.max_connections``, etc. The ``prefix`` argument
specifies the prefix used to filter keys. Each key is mapped to a
corresponding keyword argument on the :py:class:`redis.ConnectionPool`
constructor.
Supported keys:
* ``url`` (required): a URL like ``redis://localhost/0``.
* ``max_connections``: an integer maximum number of connections in the pool
* ``socket_connect_timeout``: a timespan of how long to wait for sockets
to connect. e.g. ``200 milliseconds``.
* ``socket_timeout``: a timespan of how long to wait for socket operations,
e.g. ``200 milliseconds``.
"""
assert prefix.endswith(".")
config_prefix = prefix[:-1]
cfg = config.parse_config(
app_config,
{
config_prefix: {
"url":
config.String,
"max_connections":
config.Optional(config.Integer, default=None),
"socket_connect_timeout":
config.Optional(config.Timespan, default=None),
"socket_timeout":
config.Optional(config.Timespan, default=None),
}
},
)
options = getattr(cfg, config_prefix)
if options.max_connections is not None:
kwargs.setdefault("max_connections", options.max_connections)
if options.socket_connect_timeout is not None:
kwargs.setdefault("socket_connect_timeout",
options.socket_connect_timeout.total_seconds())
if options.socket_timeout is not None:
kwargs.setdefault("socket_timeout",
options.socket_timeout.total_seconds())
return redis.BlockingConnectionPool.from_url(options.url, **kwargs)
def make_server(server_config, listener, app):
# pylint: disable=maybe-no-member
cfg = config.parse_config(
server_config, {
"max_concurrency": config.Integer,
"stop_timeout": config.Optional(config.Integer, default=0),
})
pool = Pool(size=cfg.max_concurrency)
server = GeventServer(
processor=app,
listener=listener,
spawn=pool,
)
server.stop_timeout = cfg.stop_timeout
return server
def make_processor(app_config):
cfg = config.parse_config(app_config, {
"city_db_path": config.String,
})
metrics_client = make_metrics_client(app_config)
agent = diagnostics.DiagnosticsAgent()
agent.register(diagnostics.LoggingDiagnosticsObserver())
agent.register(diagnostics.MetricsDiagnosticsObserver(metrics_client))
handler = Handler(cfg.city_db_path)
processor = GeoipService.ContextProcessor(handler)
event_handler = BaseplateProcessorEventHandler(logger, agent)
processor.setEventHandler(event_handler)
return processor
def make_processor(app_config):
cfg = config.parse_config(app_config, {
"real_random": config.Boolean,
})
metrics_client = make_metrics_client(app_config)
agent = diagnostics.DiagnosticsAgent()
agent.register(diagnostics.LoggingDiagnosticsObserver())
agent.register(diagnostics.MetricsDiagnosticsObserver(metrics_client))
handler = Handler(real_random=cfg.real_random)
processor = ToyService.ContextProcessor(handler)
event_handler = BaseplateProcessorEventHandler(logger, agent)
processor.setEventHandler(event_handler)
return processor
def make_plugin(application, app_config):
http_config = config.parse_config(app_config, {
"endpoint": config.String,
"hmac_secret": config.String,
})
root = resource.Resource()
harold = resource.Resource()
root.putChild('harold', harold)
site = server.Site(root)
site.noisy = False
site.displayTracebacks = False
endpoint = serverFromString(reactor, http_config.endpoint)
service = internet.StreamServerEndpointService(endpoint, site)
service.setServiceParent(application)
return HttpPlugin(harold, http_config.hmac_secret)
def engine_from_config(app_config, secrets=None, prefix="database."):
"""Make an :py:class:`~sqlalchemy.engine.Engine` from a configuration dictionary.
The keys useful to :py:func:`engine_from_config` should be prefixed, e.g.
``database.url``, etc. The ``prefix`` argument specifies the prefix used to
filter keys.
Supported keys:
* ``url``: the connection URL to the database, passed to
:py:func:`~sqlalchemy.engine.url.make_url` to create the
:py:class:`~sqlalchemy.engine.url.URL` used to connect to the database.
* ``credentials_secret`` (optional): the key used to retrieve the database
credentials from ``secrets`` as a :py:class:`~baseplate.secrets.CredentialSecret`.
If this is supplied, any credentials given in ``url`` we be replaced by
these.
"""
assert prefix.endswith(".")
config_prefix = prefix[:-1]
cfg = config.parse_config(
app_config,
{
config_prefix: {
"url": config.String,
"credentials_secret": config.Optional(config.String),
}
},
)
options = getattr(cfg, config_prefix)
url = make_url(options.url)
if options.credentials_secret:
if not secrets:
raise TypeError(
"'secrets' is a required argument to 'engine_from_config' "
"if 'credentials_secret' is set"
)
credentials = secrets.get_credentials(options.credentials_secret)
url.username = credentials.username
url.password = credentials.password
return create_engine(url)
def make_plugin(application, app_config):
slack_config = config.parse_config(app_config, {
"token": config.String,
})
api_client = SlackWebClient(slack_config.token)
endpoint = SlackEndpoint(api_client)
plugin = SlackPlugin(api_client)
factory = SlackClientFactory(
plugin=plugin,
useragent="Harold ([email protected])",
)
factory.setProtocolOptions(
autoPingInterval=5,
autoPingTimeout=10,
)
service = ClientService(endpoint, factory)
service.setServiceParent(application)
return plugin
def test_simple_config(self):
result = config.parse_config(
self.config, {
"simple": config.String,
"foo": {
"bar": config.Integer,
},
"noo": {
"bar": config.Optional(config.String, default=""),
},
"deep": {
"so": {
"deep": config.String,
},
},
})
self.assertEqual(result.simple, "oink")
self.assertEqual(result.foo.bar, 33)
self.assertEqual(result.noo.bar, "")
self.assertEqual(result.deep.so.deep, "very")
def make_wsgi_app(app_config):
cfg = config.parse_config(app_config, {
"session": {
"secret": config.Base64,
}
})
# configure pyramid
configurator = Configurator(settings=app_config)
configurator.include("pyramid_jinja2")
configurator.set_default_csrf_options(require_csrf=True)
configurator.set_session_factory(SignedCookieSessionFactory(cfg.session.secret))
authn_policy = RemoteUserAuthenticationPolicy(environ_key="HTTP_AUTHENTICATED_USER")
authz_policy = ACLAuthorizationPolicy()
configurator.set_authentication_policy(authn_policy)
configurator.set_authorization_policy(authz_policy)
configurator.add_request_method(get_authenticated_user, "user", reify=True)
configurator.add_static_view(name="static", path="condor:static/")
configurator.add_route("home", "/")
configurator.add_route("polls", "/polls")
configurator.add_route("poll", "/polls/{id:\d+}")
configurator.scan("condor.views")
# configure baseplate
metrics_client = make_metrics_client(app_config)
baseplate = Baseplate()
baseplate.configure_logging()
baseplate.configure_metrics(metrics_client)
engine = engine_from_config(app_config, prefix="database.")
baseplate.add_to_context("db", SQLAlchemySessionContextFactory(engine))
baseplate_configurator = BaseplateConfigurator(baseplate)
configurator.include(baseplate_configurator.includeme)
return configurator.make_wsgi_app()
def make_processor(app_config): # pragma: nocover
cfg = config.parse_config(app_config, {
"activity": {
"window": config.Timespan,
},
"tracing": {
"endpoint": config.Optional(config.Endpoint),
"service_name": config.String,
},
"redis": {
"url": config.String,
"max_connections": config.Optional(config.Integer, default=100),
},
})
metrics_client = metrics_client_from_config(app_config)
tracing_client = tracing_client_from_config(app_config)
error_reporter = error_reporter_from_config(app_config, __name__)
redis_pool = redis.BlockingConnectionPool.from_url(
cfg.redis.url,
max_connections=cfg.redis.max_connections,
timeout=0.1,
)
baseplate = Baseplate()
baseplate.configure_logging()
baseplate.configure_metrics(metrics_client)
baseplate.configure_tracing(tracing_client)
baseplate.configure_error_reporting(error_reporter)
baseplate.add_to_context("redis", RedisContextFactory(redis_pool))
counter = ActivityCounter(cfg.activity.window.total_seconds())
handler = Handler(counter=counter)
processor = ActivityService.ContextProcessor(handler)
event_handler = BaseplateProcessorEventHandler(logger, baseplate)
processor.setEventHandler(event_handler)
return processor
def make_processor(app_config):
cfg = config.parse_config(app_config, {
'redis_endpoint': config.String,
'fuzz_threshold': config.Integer,
'activity_window': config.Integer,
})
metrics_client = make_metrics_client(app_config)
agent = diagnostics.DiagnosticsAgent()
agent.register(diagnostics.LoggingDiagnosticsObserver())
agent.register(diagnostics.MetricsDiagnosticsObserver(metrics_client))
handler = Handler(
redis_endpoint=cfg.redis_endpoint,
fuzz_threshold=cfg.fuzz_threshold,
activity_window=cfg.activity_window,
)
processor = ActivityService.ContextProcessor(handler)
event_handler = BaseplateProcessorEventHandler(logger, agent)
processor.setEventHandler(event_handler)
return processor
def test_simple_config(self):
result = config.parse_config(self.config, {
"simple": config.String,
"foo": {
"bar": config.Integer,
},
"noo": {
"bar": config.String,
},
"deep": {
"so": {
"deep": config.String,
},
},
})
self.assertEqual(result.simple, "oink")
self.assertEqual(result.foo.bar, 33)
self.assertEqual(result.noo.bar, "")
self.assertEqual(result.deep.so.deep, "very")
def configure_context(self, app_config: config.RawConfig, context_spec: Dict) -> None:
"""Add a number of objects to each request's context object.
Configure and attach multiple clients to the :term:`context object` in
one place. This takes a full configuration spec like
:py:func:`baseplate.config.parse_config` and will attach the specified
structure onto the context object each request.
For example, a configuration like::
baseplate = Baseplate()
baseplate.configure_context(app_config, {
"cfg": {
"doggo_is_good": config.Boolean,
},
"cache": MemcachedClient(),
"cassandra": {
"foo": CassandraClient(),
"bar": CassandraClient(),
},
})
would build a context object that could be used like::
assert context.cfg.doggo_is_good == True
context.cache.get("example")
context.cassandra.foo.execute()
:param config: The raw stringy configuration dictionary.
:param context_spec: A specification of what the config should look
like. This should only contain context clients and nested dictionaries.
Unrelated configuration values should not be included.
"""
cfg = config.parse_config(app_config, context_spec)
self._context_config.update(cfg)
def make_app(raw_config):
cfg = config.parse_config(raw_config, CONFIG_SPEC)
metrics_client = make_metrics_client(raw_config)
dispatcher = MessageDispatcher(metrics=metrics_client)
source = MessageSource(
config=cfg.amqp,
)
app = SocketServer(
metrics=metrics_client,
dispatcher=dispatcher,
mac_secret=cfg.web.mac_secret,
ping_interval=cfg.web.ping_interval,
)
source.message_handler = dispatcher.on_message_received
app.status_publisher = source.send_message
gevent.spawn(source.pump_messages)
return app
def cluster_from_config(app_config, prefix="cassandra.", **kwargs):
"""Make a Cluster from a configuration dictionary.
The keys useful to :py:func:`cluster_from_config` should be prefixed, e.g.
``cassandra.contact_points`` etc. The ``prefix`` argument specifies the
prefix used to filter keys. Each key is mapped to a corresponding keyword
argument on the :py:class:`~cassandra.cluster.Cluster` constructor. Any
keyword arguments given to this function will be passed through to the
:py:class:`~cassandra.cluster.Cluster` constructor. Keyword arguments take
precedence over the configuration file.
Supported keys:
* ``contact_points`` (required): comma delimited list of contact points to
try connecting for cluster discovery
* ``port``: The server-side port to open connections to.
"""
assert prefix.endswith(".")
config_prefix = prefix[:-1]
cfg = config.parse_config(
app_config,
{
config_prefix: {
"contact_points": config.TupleOf(config.String),
"port": config.Optional(config.Integer, default=None),
}
},
)
options = getattr(cfg, config_prefix)
if options.port:
kwargs.setdefault("port", options.port)
return Cluster(options.contact_points, **kwargs)
def test_spec_contains_invalid_object(self):
with self.assertRaises(AssertionError):
config.parse_config(self.config, {
"tree_people": 37,
})
def test_dot_in_key(self):
with self.assertRaises(AssertionError):
config.parse_config(self.config, {
"foo.bar": {},
})
