def make_processor(app_config): # pragma: nocover cfg = config.parse_config(app_config, { "activity": { "window": config.Timespan, "fuzz_threshold": config.Integer, }, "redis": { "url": config.String, "max_connections": config.Optional(config.Integer, default=100), }, }) metrics_client = make_metrics_client(app_config) redis_pool = redis.BlockingConnectionPool.from_url( cfg.redis.url, max_connections=cfg.redis.max_connections, timeout=0.1, ) baseplate = Baseplate() baseplate.configure_logging() baseplate.configure_metrics(metrics_client) baseplate.add_to_context("redis", RedisContextFactory(redis_pool)) counter = ActivityCounter(cfg.activity.window.total_seconds()) handler = Handler( fuzz_threshold=cfg.activity.fuzz_threshold, counter=counter, ) processor = ActivityService.ContextProcessor(handler) event_handler = BaseplateProcessorEventHandler(logger, baseplate) processor.setEventHandler(event_handler) return processor
def make_wsgi_app(app_config): cfg = config.parse_config(app_config) signer = MessageSigner(cfg.ads_tracking.click_secret) metrics_client = make_metrics_client(app_config) baseplate = Baseplate() baseplate.configure_logging() baseplate.configure_metrics(metrics_client) baseplate.add_to_context("events", events.EventQueue("production")) baseplate.add_to_context("events_test", events.EventQueue("test")) configurator = Configurator(settings=app_config) baseplate_configurator = BaseplateConfigurator(baseplate) configurator.include(baseplate_configurator.includeme) controller = TrackingService( signer=signer, ) configurator.add_route("health", "/health", request_method="GET") configurator.add_view( controller.is_healthy, route_name="health", renderer="json") configurator.add_route("click", "/click", request_method="GET") configurator.add_view( controller.track_click, route_name="click", renderer="json") return configurator.make_wsgi_app()
def make_wsgi_app(app_config): cfg = config.parse_config(app_config, { "activity": { "endpoint": config.Endpoint, }, }) metrics_client = make_metrics_client(app_config) pool = ThriftConnectionPool(cfg.activity.endpoint) baseplate = Baseplate() baseplate.configure_logging() baseplate.configure_metrics(metrics_client) baseplate.add_to_context( "activity", ThriftContextFactory(pool, ActivityService.Client)) configurator = Configurator(settings=app_config) baseplate_configurator = BaseplateConfigurator(baseplate) configurator.include(baseplate_configurator.includeme) controller = ActivityGateway() configurator.add_route("health", "/health", request_method="GET") configurator.add_view(controller.is_healthy, route_name="health", renderer="json") configurator.add_route("pixel", "/{context_id:[A-Za-z0-9_]{,40}}.png", request_method="GET") configurator.add_view(controller.pixel, route_name="pixel") return configurator.make_wsgi_app()
def make_app(global_config, **settings): """Paste entry point: return a configured WSGI application.""" config = Configurator(settings=settings) keystore = {} for setting, value in settings.iteritems(): key_prefix = "key." if setting.startswith(key_prefix): key_name = setting[len(key_prefix):] key_secret = base64.b64decode(value) keystore[key_name] = key_secret allowed_origins = [ x.strip() for x in settings["allowed_origins"].split(",") if x.strip()] metrics_client = baseplate.make_metrics_client(settings) event_queue = MessageQueue("/events", max_messages=MAXIMUM_QUEUE_LENGTH, max_message_size=MAXIMUM_EVENT_SIZE) error_queue = MessageQueue("/errors", max_messages=MAXIMUM_QUEUE_LENGTH, max_message_size=MAXIMUM_EVENT_SIZE) collector = EventCollector( keystore, metrics_client, event_queue, error_queue, allowed_origins) config.add_route("v1", "/v1", request_method="POST") config.add_route("v1_options", "/v1", request_method="OPTIONS") config.add_view(collector.process_request, route_name="v1") config.add_view(collector.check_cors, route_name="v1_options") config.add_route("health", "/health") config.add_view(health_check, route_name="health", renderer="json") return config.make_wsgi_app()
def make_wsgi_app(app_config): cfg = config.parse_config(app_config) signer = MessageSigner(cfg.ads_tracking.click_secret) metrics_client = make_metrics_client(app_config) baseplate = Baseplate() baseplate.configure_logging() baseplate.configure_metrics(metrics_client) baseplate.add_to_context("events", events.EventQueue("production")) baseplate.add_to_context("events_test", events.EventQueue("test")) configurator = Configurator(settings=app_config) baseplate_configurator = BaseplateConfigurator(baseplate) configurator.include(baseplate_configurator.includeme) controller = TrackingService(signer=signer, ) configurator.add_route("health", "/health", request_method="GET") configurator.add_view(controller.is_healthy, route_name="health", renderer="json") configurator.add_route("click", "/click", request_method="GET") configurator.add_view(controller.track_click, route_name="click", renderer="json") return configurator.make_wsgi_app()
def make_app(global_config, **settings): """Paste entry point: return a configured WSGI application.""" config = Configurator(settings=settings) keystore = {} for setting, value in settings.iteritems(): key_prefix = "key." if setting.startswith(key_prefix): key_name = setting[len(key_prefix):] key_secret = base64.b64decode(value) keystore[key_name] = key_secret allowed_origins = [ x.strip() for x in settings["allowed_origins"].split(",") if x.strip() ] metrics_client = baseplate.make_metrics_client(settings) event_queue = MessageQueue("/events", max_messages=MAXIMUM_QUEUE_LENGTH, max_message_size=MAXIMUM_EVENT_SIZE) error_queue = MessageQueue("/errors", max_messages=MAXIMUM_QUEUE_LENGTH, max_message_size=MAXIMUM_EVENT_SIZE) collector = EventCollector(keystore, metrics_client, event_queue, error_queue, allowed_origins) config.add_route("v1", "/v1", request_method="POST") config.add_route("v1_options", "/v1", request_method="OPTIONS") config.add_view(collector.process_request, route_name="v1") config.add_view(collector.check_cors, route_name="v1_options") config.add_route("health", "/health") config.add_view(health_check, route_name="health", renderer="json") return config.make_wsgi_app()
def make_app(raw_config): cfg = config.parse_config(raw_config, CONFIG_SPEC) metrics_client = make_metrics_client(raw_config) dispatcher = MessageDispatcher(metrics=metrics_client) source = MessageSource(config=cfg.amqp, ) app = SocketServer( metrics=metrics_client, dispatcher=dispatcher, mac_secret=cfg.web.mac_secret, ping_interval=cfg.web.ping_interval, admin_auth=cfg.web.admin_auth, conn_shed_rate=cfg.web.conn_shed_rate, ) # register SIGUSR2 to trigger app quiescing, # useful if app processes are behind # a process manager like einhorn. def _handle_quiesce_signal(_, frame): app._quiesce({}, bypass_auth=True) signal.signal(signal.SIGUSR2, _handle_quiesce_signal) signal.siginterrupt(signal.SIGUSR2, False) source.message_handler = dispatcher.on_message_received app.status_publisher = source.send_message gevent.spawn(source.pump_messages) return app
def main(): """Run a consumer. Two environment variables are expected: * CONFIG_URI: A PasteDeploy URI pointing at the configuration for the application. * QUEUE: The name of the queue to consume (currently one of "events" or "errors"). """ config_uri = os.environ["CONFIG_URI"] config = paste.deploy.loadwsgi.appconfig(config_uri) logging.config.fileConfig(config["__file__"]) queue_name = os.environ["QUEUE"] queue = MessageQueue("/" + queue_name, max_messages=MAXIMUM_QUEUE_LENGTH, max_message_size=MAXIMUM_EVENT_SIZE) metrics_client = baseplate.make_metrics_client(config) topic_name = config["topic." + queue_name] producer_options = { "codec": CODEC_GZIP, "batch_send_every_n": 20, "batch_send_every_t": 0.01, # 10 milliseconds } while True: try: kafka_client = KafkaClient(config["kafka_brokers"]) kafka_producer = SimpleProducer(kafka_client, **producer_options) except KafkaError as exc: _LOG.warning("could not connect: %s", exc) metrics_client.counter("injector.connection_error").increment() time.sleep(_RETRY_DELAY) continue while True: message = queue.get() for retry in itertools.count(): try: kafka_producer.send_messages(topic_name, message) except KafkaError as exc: _LOG.warning("failed to send message: %s", exc) metrics_client.counter("injector.error").increment() time.sleep(_RETRY_DELAY) else: metrics_client.counter("collected.injector").increment() break kafka_producer.stop()
def make_processor(app_config): cfg = config.parse_config(app_config, { "real_random": config.Boolean, }) metrics_client = make_metrics_client(app_config) agent = diagnostics.DiagnosticsAgent() agent.register(diagnostics.LoggingDiagnosticsObserver()) agent.register(diagnostics.MetricsDiagnosticsObserver(metrics_client)) handler = Handler(real_random=cfg.real_random) processor = ToyService.ContextProcessor(handler) event_handler = BaseplateProcessorEventHandler(logger, agent) processor.setEventHandler(event_handler) return processor
def make_processor(app_config): cfg = config.parse_config(app_config, { "city_db_path": config.String, }) metrics_client = make_metrics_client(app_config) agent = diagnostics.DiagnosticsAgent() agent.register(diagnostics.LoggingDiagnosticsObserver()) agent.register(diagnostics.MetricsDiagnosticsObserver(metrics_client)) handler = Handler(cfg.city_db_path) processor = GeoipService.ContextProcessor(handler) event_handler = BaseplateProcessorEventHandler(logger, agent) processor.setEventHandler(event_handler) return processor
def make_wsgi_app(app_config): cfg = config.parse_config(app_config, { "session": { "secret": config.Base64, } }) # configure pyramid configurator = Configurator(settings=app_config) configurator.include("pyramid_jinja2") configurator.set_default_csrf_options(require_csrf=True) configurator.set_session_factory(SignedCookieSessionFactory(cfg.session.secret)) authn_policy = RemoteUserAuthenticationPolicy(environ_key="HTTP_AUTHENTICATED_USER") authz_policy = ACLAuthorizationPolicy() configurator.set_authentication_policy(authn_policy) configurator.set_authorization_policy(authz_policy) configurator.add_request_method(get_authenticated_user, "user", reify=True) configurator.add_static_view(name="static", path="condor:static/") configurator.add_route("home", "/") configurator.add_route("polls", "/polls") configurator.add_route("poll", "/polls/{id:\d+}") configurator.scan("condor.views") # configure baseplate metrics_client = make_metrics_client(app_config) baseplate = Baseplate() baseplate.configure_logging() baseplate.configure_metrics(metrics_client) engine = engine_from_config(app_config, prefix="database.") baseplate.add_to_context("db", SQLAlchemySessionContextFactory(engine)) baseplate_configurator = BaseplateConfigurator(baseplate) configurator.include(baseplate_configurator.includeme) return configurator.make_wsgi_app()
def make_processor(app_config): cfg = config.parse_config(app_config, { 'redis_endpoint': config.String, 'fuzz_threshold': config.Integer, 'activity_window': config.Integer, }) metrics_client = make_metrics_client(app_config) agent = diagnostics.DiagnosticsAgent() agent.register(diagnostics.LoggingDiagnosticsObserver()) agent.register(diagnostics.MetricsDiagnosticsObserver(metrics_client)) handler = Handler( redis_endpoint=cfg.redis_endpoint, fuzz_threshold=cfg.fuzz_threshold, activity_window=cfg.activity_window, ) processor = ActivityService.ContextProcessor(handler) event_handler = BaseplateProcessorEventHandler(logger, agent) processor.setEventHandler(event_handler) return processor
def make_wsgi_app(app_config): cfg = config.parse_config(app_config, { # TODO: add your config spec here }) metrics_client = make_metrics_client(app_config) baseplate = Baseplate() baseplate.configure_logging() baseplate.configure_metrics(metrics_client) configurator = Configurator(settings=app_config) baseplate_configurator = BaseplateConfigurator(baseplate) configurator.include(baseplate_configurator.includeme) controller = {{ cookiecutter.service_name }}() configurator.add_route("health", "/health", request_method="GET") configurator.add_view( controller.is_healthy, route_name="health", renderer="json") # TODO: add more routes and views here return configurator.make_wsgi_app()
def make_app(raw_config): cfg = config.parse_config(raw_config, CONFIG_SPEC) metrics_client = make_metrics_client(raw_config) dispatcher = MessageDispatcher(metrics=metrics_client) source = MessageSource( config=cfg.amqp, ) app = SocketServer( metrics=metrics_client, dispatcher=dispatcher, mac_secret=cfg.web.mac_secret, ping_interval=cfg.web.ping_interval, ) source.message_handler = dispatcher.on_message_received app.status_publisher = source.send_message gevent.spawn(source.pump_messages) return app
def main(): """Run a consumer. Two environment variables are expected: * CONFIG_URI: A PasteDeploy URI pointing at the configuration for the application. * QUEUE: The name of the queue to consume (currently one of "events" or "errors"). """ config_uri = os.environ["CONFIG_URI"] config = paste.deploy.loadwsgi.appconfig(config_uri) logging.config.fileConfig(config["__file__"]) queue_name = os.environ["QUEUE"] queue = MessageQueue( "/" + queue_name, max_messages=MAXIMUM_QUEUE_LENGTH[queue_name], max_message_size=MAXIMUM_MESSAGE_SIZE[queue_name], ) metrics_client = baseplate.make_metrics_client(config) topic_name = config["topic." + queue_name] # Details at http://kafka-python.readthedocs.org/en/1.0.2/apidoc/KafkaProducer.html producer_options = { "compression_type": 'gzip', "batch_size": 20, "linger_ms": 10, "retries": int(config["kafka_retries"]), "retry_backoff_ms": _RETRY_DELAY_SECS * 1000 } def producer_error_cb(msg, queue): def requeue_msg(exc): _LOG.warning("failed to send message=%s due to error=%s", msg, exc) metrics_client.counter("injector.error").increment() queue.put(msg) return requeue_msg def producer_success_cb(success_val): metrics_client.counter("collected.injector").increment() while True: try: kafka_brokers = [ broker.strip() for broker in config['kafka_brokers'].split(',') ] kafka_producer = KafkaProducer(bootstrap_servers=kafka_brokers, **producer_options) except KafkaError as exc: _LOG.warning("could not connect: %s", exc) metrics_client.counter("injector.connection_error").increment() time.sleep(_RETRY_DELAY_SECS) continue process_queue(queue, topic_name, kafka_producer, producer_success_cb, producer_error_cb, metrics_client=metrics_client) kafka_producer.stop()
def main(): """Run a consumer. Two environment variables are expected: * CONFIG_URI: A PasteDeploy URI pointing at the configuration for the application. * QUEUE: The name of the queue to consume (currently one of "events" or "errors"). """ config_uri = os.environ["CONFIG_URI"] config = paste.deploy.loadwsgi.appconfig(config_uri) logging.config.fileConfig(config["__file__"]) queue_name = os.environ["QUEUE"] queue = MessageQueue( "/" + queue_name, max_messages=MAXIMUM_QUEUE_LENGTH[queue_name], max_message_size=MAXIMUM_MESSAGE_SIZE[queue_name], ) metrics_client = baseplate.make_metrics_client(config) topic_name = config["topic." + queue_name] # Details at http://kafka-python.readthedocs.org/en/1.0.2/apidoc/KafkaProducer.html producer_options = { "compression_type": 'gzip', "batch_size": 20, "linger_ms": 10, "retries": int(config["kafka_retries"]), "retry_backoff_ms": _RETRY_DELAY_SECS * 1000 } def producer_error_cb(msg, queue): def requeue_msg(exc): _LOG.warning("failed to send message=%s due to error=%s", msg, exc) metrics_client.counter("injector.error").increment() queue.put(msg) return requeue_msg def producer_success_cb(success_val): metrics_client.counter("collected.injector").increment() while True: try: kafka_brokers = [broker.strip() for broker in config['kafka_brokers'].split(',')] kafka_producer = KafkaProducer(bootstrap_servers=kafka_brokers, **producer_options) except KafkaError as exc: _LOG.warning("could not connect: %s", exc) metrics_client.counter("injector.connection_error").increment() time.sleep(_RETRY_DELAY_SECS) continue process_queue(queue, topic_name, kafka_producer, producer_success_cb, producer_error_cb, metrics_client=metrics_client) kafka_producer.stop()