def make_processor(app_config): # pragma: nocover
cfg = config.parse_config(app_config, {
"activity": {
"window": config.Timespan,
"fuzz_threshold": config.Integer,
},
"redis": {
"url": config.String,
"max_connections": config.Optional(config.Integer, default=100),
},
})
metrics_client = make_metrics_client(app_config)
redis_pool = redis.BlockingConnectionPool.from_url(
cfg.redis.url,
max_connections=cfg.redis.max_connections,
timeout=0.1,
)
baseplate = Baseplate()
baseplate.configure_logging()
baseplate.configure_metrics(metrics_client)
baseplate.add_to_context("redis", RedisContextFactory(redis_pool))
counter = ActivityCounter(cfg.activity.window.total_seconds())
handler = Handler(
fuzz_threshold=cfg.activity.fuzz_threshold,
counter=counter,
)
processor = ActivityService.ContextProcessor(handler)
event_handler = BaseplateProcessorEventHandler(logger, baseplate)
processor.setEventHandler(event_handler)
return processor
def make_wsgi_app(app_config):
cfg = config.parse_config(app_config)
signer = MessageSigner(cfg.ads_tracking.click_secret)
metrics_client = make_metrics_client(app_config)
baseplate = Baseplate()
baseplate.configure_logging()
baseplate.configure_metrics(metrics_client)
baseplate.add_to_context("events", events.EventQueue("production"))
baseplate.add_to_context("events_test", events.EventQueue("test"))
configurator = Configurator(settings=app_config)
baseplate_configurator = BaseplateConfigurator(baseplate)
configurator.include(baseplate_configurator.includeme)
controller = TrackingService(
signer=signer,
)
configurator.add_route("health", "/health", request_method="GET")
configurator.add_view(
controller.is_healthy, route_name="health", renderer="json")
configurator.add_route("click", "/click", request_method="GET")
configurator.add_view(
controller.track_click, route_name="click", renderer="json")
return configurator.make_wsgi_app()
def make_wsgi_app(app_config):
cfg = config.parse_config(app_config, {
"activity": {
"endpoint": config.Endpoint,
},
})
metrics_client = make_metrics_client(app_config)
pool = ThriftConnectionPool(cfg.activity.endpoint)
baseplate = Baseplate()
baseplate.configure_logging()
baseplate.configure_metrics(metrics_client)
baseplate.add_to_context(
"activity", ThriftContextFactory(pool, ActivityService.Client))
configurator = Configurator(settings=app_config)
baseplate_configurator = BaseplateConfigurator(baseplate)
configurator.include(baseplate_configurator.includeme)
controller = ActivityGateway()
configurator.add_route("health", "/health", request_method="GET")
configurator.add_view(controller.is_healthy,
route_name="health",
renderer="json")
configurator.add_route("pixel",
"/{context_id:[A-Za-z0-9_]{,40}}.png",
request_method="GET")
configurator.add_view(controller.pixel, route_name="pixel")
return configurator.make_wsgi_app()
def make_app(global_config, **settings):
"""Paste entry point: return a configured WSGI application."""
config = Configurator(settings=settings)
keystore = {}
for setting, value in settings.iteritems():
key_prefix = "key."
if setting.startswith(key_prefix):
key_name = setting[len(key_prefix):]
key_secret = base64.b64decode(value)
keystore[key_name] = key_secret
allowed_origins = [
x.strip() for x in settings["allowed_origins"].split(",") if x.strip()]
metrics_client = baseplate.make_metrics_client(settings)
event_queue = MessageQueue("/events",
max_messages=MAXIMUM_QUEUE_LENGTH, max_message_size=MAXIMUM_EVENT_SIZE)
error_queue = MessageQueue("/errors",
max_messages=MAXIMUM_QUEUE_LENGTH, max_message_size=MAXIMUM_EVENT_SIZE)
collector = EventCollector(
keystore, metrics_client, event_queue, error_queue, allowed_origins)
config.add_route("v1", "/v1", request_method="POST")
config.add_route("v1_options", "/v1", request_method="OPTIONS")
config.add_view(collector.process_request, route_name="v1")
config.add_view(collector.check_cors, route_name="v1_options")
config.add_route("health", "/health")
config.add_view(health_check, route_name="health", renderer="json")
return config.make_wsgi_app()
def make_wsgi_app(app_config):
cfg = config.parse_config(app_config)
signer = MessageSigner(cfg.ads_tracking.click_secret)
metrics_client = make_metrics_client(app_config)
baseplate = Baseplate()
baseplate.configure_logging()
baseplate.configure_metrics(metrics_client)
baseplate.add_to_context("events", events.EventQueue("production"))
baseplate.add_to_context("events_test", events.EventQueue("test"))
configurator = Configurator(settings=app_config)
baseplate_configurator = BaseplateConfigurator(baseplate)
configurator.include(baseplate_configurator.includeme)
controller = TrackingService(signer=signer, )
configurator.add_route("health", "/health", request_method="GET")
configurator.add_view(controller.is_healthy,
route_name="health",
renderer="json")
configurator.add_route("click", "/click", request_method="GET")
configurator.add_view(controller.track_click,
route_name="click",
renderer="json")
return configurator.make_wsgi_app()
def make_app(global_config, **settings):
"""Paste entry point: return a configured WSGI application."""
config = Configurator(settings=settings)
keystore = {}
for setting, value in settings.iteritems():
key_prefix = "key."
if setting.startswith(key_prefix):
key_name = setting[len(key_prefix):]
key_secret = base64.b64decode(value)
keystore[key_name] = key_secret
allowed_origins = [
x.strip() for x in settings["allowed_origins"].split(",") if x.strip()
]
metrics_client = baseplate.make_metrics_client(settings)
event_queue = MessageQueue("/events",
max_messages=MAXIMUM_QUEUE_LENGTH,
max_message_size=MAXIMUM_EVENT_SIZE)
error_queue = MessageQueue("/errors",
max_messages=MAXIMUM_QUEUE_LENGTH,
max_message_size=MAXIMUM_EVENT_SIZE)
collector = EventCollector(keystore, metrics_client, event_queue,
error_queue, allowed_origins)
config.add_route("v1", "/v1", request_method="POST")
config.add_route("v1_options", "/v1", request_method="OPTIONS")
config.add_view(collector.process_request, route_name="v1")
config.add_view(collector.check_cors, route_name="v1_options")
config.add_route("health", "/health")
config.add_view(health_check, route_name="health", renderer="json")
return config.make_wsgi_app()
def make_app(raw_config):
cfg = config.parse_config(raw_config, CONFIG_SPEC)
metrics_client = make_metrics_client(raw_config)
dispatcher = MessageDispatcher(metrics=metrics_client)
source = MessageSource(config=cfg.amqp, )
app = SocketServer(
metrics=metrics_client,
dispatcher=dispatcher,
mac_secret=cfg.web.mac_secret,
ping_interval=cfg.web.ping_interval,
admin_auth=cfg.web.admin_auth,
conn_shed_rate=cfg.web.conn_shed_rate,
)
# register SIGUSR2 to trigger app quiescing,
# useful if app processes are behind
# a process manager like einhorn.
def _handle_quiesce_signal(_, frame):
app._quiesce({}, bypass_auth=True)
signal.signal(signal.SIGUSR2, _handle_quiesce_signal)
signal.siginterrupt(signal.SIGUSR2, False)
source.message_handler = dispatcher.on_message_received
app.status_publisher = source.send_message
gevent.spawn(source.pump_messages)
return app
def main():
"""Run a consumer.
Two environment variables are expected:
* CONFIG_URI: A PasteDeploy URI pointing at the configuration for the
application.
* QUEUE: The name of the queue to consume (currently one of "events" or
"errors").
"""
config_uri = os.environ["CONFIG_URI"]
config = paste.deploy.loadwsgi.appconfig(config_uri)
logging.config.fileConfig(config["__file__"])
queue_name = os.environ["QUEUE"]
queue = MessageQueue("/" + queue_name,
max_messages=MAXIMUM_QUEUE_LENGTH,
max_message_size=MAXIMUM_EVENT_SIZE)
metrics_client = baseplate.make_metrics_client(config)
topic_name = config["topic." + queue_name]
producer_options = {
"codec": CODEC_GZIP,
"batch_send_every_n": 20,
"batch_send_every_t": 0.01, # 10 milliseconds
}
while True:
try:
kafka_client = KafkaClient(config["kafka_brokers"])
kafka_producer = SimpleProducer(kafka_client, **producer_options)
except KafkaError as exc:
_LOG.warning("could not connect: %s", exc)
metrics_client.counter("injector.connection_error").increment()
time.sleep(_RETRY_DELAY)
continue
while True:
message = queue.get()
for retry in itertools.count():
try:
kafka_producer.send_messages(topic_name, message)
except KafkaError as exc:
_LOG.warning("failed to send message: %s", exc)
metrics_client.counter("injector.error").increment()
time.sleep(_RETRY_DELAY)
else:
metrics_client.counter("collected.injector").increment()
break
kafka_producer.stop()
def main():
"""Run a consumer.
Two environment variables are expected:
* CONFIG_URI: A PasteDeploy URI pointing at the configuration for the
application.
* QUEUE: The name of the queue to consume (currently one of "events" or
"errors").
"""
config_uri = os.environ["CONFIG_URI"]
config = paste.deploy.loadwsgi.appconfig(config_uri)
logging.config.fileConfig(config["__file__"])
queue_name = os.environ["QUEUE"]
queue = MessageQueue("/" + queue_name,
max_messages=MAXIMUM_QUEUE_LENGTH, max_message_size=MAXIMUM_EVENT_SIZE)
metrics_client = baseplate.make_metrics_client(config)
topic_name = config["topic." + queue_name]
producer_options = {
"codec": CODEC_GZIP,
"batch_send_every_n": 20,
"batch_send_every_t": 0.01, # 10 milliseconds
}
while True:
try:
kafka_client = KafkaClient(config["kafka_brokers"])
kafka_producer = SimpleProducer(kafka_client, **producer_options)
except KafkaError as exc:
_LOG.warning("could not connect: %s", exc)
metrics_client.counter("injector.connection_error").increment()
time.sleep(_RETRY_DELAY)
continue
while True:
message = queue.get()
for retry in itertools.count():
try:
kafka_producer.send_messages(topic_name, message)
except KafkaError as exc:
_LOG.warning("failed to send message: %s", exc)
metrics_client.counter("injector.error").increment()
time.sleep(_RETRY_DELAY)
else:
metrics_client.counter("collected.injector").increment()
break
kafka_producer.stop()
def make_processor(app_config):
cfg = config.parse_config(app_config, {
"real_random": config.Boolean,
})
metrics_client = make_metrics_client(app_config)
agent = diagnostics.DiagnosticsAgent()
agent.register(diagnostics.LoggingDiagnosticsObserver())
agent.register(diagnostics.MetricsDiagnosticsObserver(metrics_client))
handler = Handler(real_random=cfg.real_random)
processor = ToyService.ContextProcessor(handler)
event_handler = BaseplateProcessorEventHandler(logger, agent)
processor.setEventHandler(event_handler)
return processor
def make_processor(app_config):
cfg = config.parse_config(app_config, {
"city_db_path": config.String,
})
metrics_client = make_metrics_client(app_config)
agent = diagnostics.DiagnosticsAgent()
agent.register(diagnostics.LoggingDiagnosticsObserver())
agent.register(diagnostics.MetricsDiagnosticsObserver(metrics_client))
handler = Handler(cfg.city_db_path)
processor = GeoipService.ContextProcessor(handler)
event_handler = BaseplateProcessorEventHandler(logger, agent)
processor.setEventHandler(event_handler)
return processor
def make_wsgi_app(app_config):
cfg = config.parse_config(app_config, {
"session": {
"secret": config.Base64,
}
})
# configure pyramid
configurator = Configurator(settings=app_config)
configurator.include("pyramid_jinja2")
configurator.set_default_csrf_options(require_csrf=True)
configurator.set_session_factory(SignedCookieSessionFactory(cfg.session.secret))
authn_policy = RemoteUserAuthenticationPolicy(environ_key="HTTP_AUTHENTICATED_USER")
authz_policy = ACLAuthorizationPolicy()
configurator.set_authentication_policy(authn_policy)
configurator.set_authorization_policy(authz_policy)
configurator.add_request_method(get_authenticated_user, "user", reify=True)
configurator.add_static_view(name="static", path="condor:static/")
configurator.add_route("home", "/")
configurator.add_route("polls", "/polls")
configurator.add_route("poll", "/polls/{id:\d+}")
configurator.scan("condor.views")
# configure baseplate
metrics_client = make_metrics_client(app_config)
baseplate = Baseplate()
baseplate.configure_logging()
baseplate.configure_metrics(metrics_client)
engine = engine_from_config(app_config, prefix="database.")
baseplate.add_to_context("db", SQLAlchemySessionContextFactory(engine))
baseplate_configurator = BaseplateConfigurator(baseplate)
configurator.include(baseplate_configurator.includeme)
return configurator.make_wsgi_app()
def make_processor(app_config):
cfg = config.parse_config(app_config, {
'redis_endpoint': config.String,
'fuzz_threshold': config.Integer,
'activity_window': config.Integer,
})
metrics_client = make_metrics_client(app_config)
agent = diagnostics.DiagnosticsAgent()
agent.register(diagnostics.LoggingDiagnosticsObserver())
agent.register(diagnostics.MetricsDiagnosticsObserver(metrics_client))
handler = Handler(
redis_endpoint=cfg.redis_endpoint,
fuzz_threshold=cfg.fuzz_threshold,
activity_window=cfg.activity_window,
)
processor = ActivityService.ContextProcessor(handler)
event_handler = BaseplateProcessorEventHandler(logger, agent)
processor.setEventHandler(event_handler)
return processor
def make_wsgi_app(app_config):
cfg = config.parse_config(app_config, {
# TODO: add your config spec here
})
metrics_client = make_metrics_client(app_config)
baseplate = Baseplate()
baseplate.configure_logging()
baseplate.configure_metrics(metrics_client)
configurator = Configurator(settings=app_config)
baseplate_configurator = BaseplateConfigurator(baseplate)
configurator.include(baseplate_configurator.includeme)
controller = {{ cookiecutter.service_name }}()
configurator.add_route("health", "/health", request_method="GET")
configurator.add_view(
controller.is_healthy, route_name="health", renderer="json")
# TODO: add more routes and views here
return configurator.make_wsgi_app()
def make_app(raw_config):
cfg = config.parse_config(raw_config, CONFIG_SPEC)
metrics_client = make_metrics_client(raw_config)
dispatcher = MessageDispatcher(metrics=metrics_client)
source = MessageSource(
config=cfg.amqp,
)
app = SocketServer(
metrics=metrics_client,
dispatcher=dispatcher,
mac_secret=cfg.web.mac_secret,
ping_interval=cfg.web.ping_interval,
)
source.message_handler = dispatcher.on_message_received
app.status_publisher = source.send_message
gevent.spawn(source.pump_messages)
return app
def main():
"""Run a consumer.
Two environment variables are expected:
* CONFIG_URI: A PasteDeploy URI pointing at the configuration for the
application.
* QUEUE: The name of the queue to consume (currently one of "events" or
"errors").
"""
config_uri = os.environ["CONFIG_URI"]
config = paste.deploy.loadwsgi.appconfig(config_uri)
logging.config.fileConfig(config["__file__"])
queue_name = os.environ["QUEUE"]
queue = MessageQueue(
"/" + queue_name,
max_messages=MAXIMUM_QUEUE_LENGTH[queue_name],
max_message_size=MAXIMUM_MESSAGE_SIZE[queue_name],
)
metrics_client = baseplate.make_metrics_client(config)
topic_name = config["topic." + queue_name]
# Details at http://kafka-python.readthedocs.org/en/1.0.2/apidoc/KafkaProducer.html
producer_options = {
"compression_type": 'gzip',
"batch_size": 20,
"linger_ms": 10,
"retries": int(config["kafka_retries"]),
"retry_backoff_ms": _RETRY_DELAY_SECS * 1000
}
def producer_error_cb(msg, queue):
def requeue_msg(exc):
_LOG.warning("failed to send message=%s due to error=%s", msg, exc)
metrics_client.counter("injector.error").increment()
queue.put(msg)
return requeue_msg
def producer_success_cb(success_val):
metrics_client.counter("collected.injector").increment()
while True:
try:
kafka_brokers = [
broker.strip() for broker in config['kafka_brokers'].split(',')
]
kafka_producer = KafkaProducer(bootstrap_servers=kafka_brokers,
**producer_options)
except KafkaError as exc:
_LOG.warning("could not connect: %s", exc)
metrics_client.counter("injector.connection_error").increment()
time.sleep(_RETRY_DELAY_SECS)
continue
process_queue(queue,
topic_name,
kafka_producer,
producer_success_cb,
producer_error_cb,
metrics_client=metrics_client)
kafka_producer.stop()
def main():
"""Run a consumer.
Two environment variables are expected:
* CONFIG_URI: A PasteDeploy URI pointing at the configuration for the
application.
* QUEUE: The name of the queue to consume (currently one of "events" or
"errors").
"""
config_uri = os.environ["CONFIG_URI"]
config = paste.deploy.loadwsgi.appconfig(config_uri)
logging.config.fileConfig(config["__file__"])
queue_name = os.environ["QUEUE"]
queue = MessageQueue(
"/" + queue_name,
max_messages=MAXIMUM_QUEUE_LENGTH[queue_name],
max_message_size=MAXIMUM_MESSAGE_SIZE[queue_name],
)
metrics_client = baseplate.make_metrics_client(config)
topic_name = config["topic." + queue_name]
# Details at http://kafka-python.readthedocs.org/en/1.0.2/apidoc/KafkaProducer.html
producer_options = {
"compression_type": 'gzip',
"batch_size": 20,
"linger_ms": 10,
"retries": int(config["kafka_retries"]),
"retry_backoff_ms": _RETRY_DELAY_SECS * 1000
}
def producer_error_cb(msg, queue):
def requeue_msg(exc):
_LOG.warning("failed to send message=%s due to error=%s", msg, exc)
metrics_client.counter("injector.error").increment()
queue.put(msg)
return requeue_msg
def producer_success_cb(success_val):
metrics_client.counter("collected.injector").increment()
while True:
try:
kafka_brokers = [broker.strip() for broker in config['kafka_brokers'].split(',')]
kafka_producer = KafkaProducer(bootstrap_servers=kafka_brokers,
**producer_options)
except KafkaError as exc:
_LOG.warning("could not connect: %s", exc)
metrics_client.counter("injector.connection_error").increment()
time.sleep(_RETRY_DELAY_SECS)
continue
process_queue(queue,
topic_name,
kafka_producer,
producer_success_cb,
producer_error_cb,
metrics_client=metrics_client)
kafka_producer.stop()
