def test_open_fail(self): # FIXME going to fail at this point due to GeoIPNull with self.assertRaises(geoip.GeoIPError): geoip.configure_geoip(filename='/i/taught/i/taw/a/putty/tat') with tempfile.NamedTemporaryFile() as temp: temp.write('Bucephalus') temp.seek(0) with self.assertRaises(geoip.GeoIPError): geoip.configure_geoip(filename=temp.name)
def init_worker( celery_app, app_config, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None, ): """ Configure the passed in celery app, usually stored in :data:`ichnaea.async.app.celery_app`. Does connection, settings and queue setup. Attaches some additional functionality to the :class:`celery.Celery` instance. This is executed inside each forked worker process. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param _db_ro: Ignored, read-only database connection isn't used. """ # make config file settings available celery_app.settings = app_config.asdict() # configure outside connections celery_app.db_rw = configure_db(app_config.get("ichnaea", "db_master"), _db=_db_rw) celery_app.raven_client = raven_client = configure_raven( app_config.get("ichnaea", "sentry_dsn"), transport="threaded", _client=_raven_client ) celery_app.redis_client = redis_client = configure_redis( app_config.get("ichnaea", "redis_url"), _client=_redis_client ) celery_app.stats_client = configure_stats(app_config.get("ichnaea", "statsd_host"), _client=_stats_client) celery_app.geoip_db = configure_geoip( app_config.get("ichnaea", "geoip_db_path"), raven_client=raven_client, _client=_geoip_db ) # configure data / export queues celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES]) celery_app.data_queues = data_queues = configure_data(redis_client) for queue in data_queues.values(): if queue.monitor_name: all_queues.add(queue.monitor_name) celery_app.export_queues = configure_export(redis_client, app_config) for queue in celery_app.export_queues.values(): if queue.monitor_name: all_queues.add(queue.monitor_name)
def init_worker(celery_app, app_config, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): """ Configure the passed in celery app, usually stored in :data:`ichnaea.async.app.celery_app`. Does connection, settings and queue setup. Attaches some additional functionality to the :class:`celery.Celery` instance. This is executed inside each forked worker process. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param _db_ro: Ignored, read-only database connection isn't used. """ # make config file settings available celery_app.settings = app_config.asdict() # configure outside connections celery_app.db_rw = configure_db(app_config.get('database', 'rw_url'), _db=_db_rw) celery_app.raven_client = raven_client = configure_raven( app_config.get('sentry', 'dsn'), transport='threaded', _client=_raven_client) celery_app.redis_client = redis_client = configure_redis( app_config.get('cache', 'cache_url'), _client=_redis_client) celery_app.stats_client = configure_stats(app_config, _client=_stats_client) celery_app.geoip_db = configure_geoip(app_config.get('geoip', 'db_path'), raven_client=raven_client, _client=_geoip_db) # configure data / export queues celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES]) celery_app.data_queues = data_queues = configure_data(redis_client) for queue in data_queues.values(): if queue.monitor_name: all_queues.add(queue.monitor_name) celery_app.export_queues = configure_export(redis_client, app_config) for queue in celery_app.export_queues.values(): if queue.monitor_name: all_queues.add(queue.monitor_name)
def main(global_config, app_config, init=False, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): configure_logging() # make config file settings available config = Configurator(settings=app_config.asdict()) # add support for pt templates config.include('pyramid_chameleon') configure_content(config) configure_service(config) # configure outside connections registry = config.registry registry.db_rw = configure_db( app_config.get('ichnaea', 'db_master'), _db=_db_rw) registry.db_ro = configure_db( app_config.get('ichnaea', 'db_slave'), _db=_db_ro) registry.raven_client = raven_client = configure_raven( app_config.get('ichnaea', 'sentry_dsn'), transport='gevent', _client=_raven_client) registry.redis_client = configure_redis( app_config.get('ichnaea', 'redis_url'), _client=_redis_client) registry.stats_client = configure_stats( app_config.get('ichnaea', 'statsd_host'), _client=_stats_client) registry.geoip_db = configure_geoip( app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client, _client=_geoip_db) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW) config.add_request_method(db_rw_session, property=True) config.add_request_method(db_ro_session, property=True) # replace json renderer with custom json variant config.add_renderer('json', customjson.Renderer()) # Should we try to initialize and establish the outbound connections? if init: # pragma: no cover registry.db_ro.ping() registry.redis_client.ping() registry.stats_client.ping() return config.make_wsgi_app()
def main(global_config, _db_master=None, _db_slave=None, **settings): config = Configurator(settings=settings) # add support for pt templates config.include('pyramid_chameleon') settings = config.registry.settings from ichnaea.content.views import configure_content from ichnaea.service import configure_service from ichnaea.heka_logging import configure_heka configure_content(config) configure_service(config) # configure databases incl. test override hooks if _db_master is None: config.registry.db_master = Database( settings['db_master'], socket=settings.get('db_master_socket'), ) else: config.registry.db_master = _db_master if _db_slave is None: config.registry.db_slave = Database( settings['db_slave'], socket=settings.get('db_slave_socket'), create=False, ) else: config.registry.db_slave = _db_slave config.registry.geoip_db = configure_geoip(config.registry.settings) config.registry.heka_client = configure_heka(config.registry.settings) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.heka_logging.heka_tween_factory', under=EXCVIEW) config.add_request_method(db_master_session, property=True) config.add_request_method(db_slave_session, property=True) # replace json renderer with decimal json variant config.add_renderer('json', decimaljson.Renderer()) return config.make_wsgi_app()
def init_worker(celery_app, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): """ Configure the passed in celery app, usually stored in :data:`ichnaea.async.app.celery_app`. Does connection, settings and queue setup. Attaches some additional functionality to the :class:`celery.Celery` instance. This is executed inside each forked worker process. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param _db_ro: Ignored, read-only database connection isn't used. """ # configure outside connections celery_app.db_rw = configure_db( celery_app.app_config.get('database', 'rw_url'), _db=_db_rw) celery_app.raven_client = raven_client = configure_raven( celery_app.app_config.get('sentry', 'dsn'), transport='threaded', _client=_raven_client) celery_app.redis_client = redis_client = configure_redis( celery_app.app_config.get('cache', 'cache_url'), _client=_redis_client) celery_app.stats_client = configure_stats( celery_app.app_config, _client=_stats_client) celery_app.geoip_db = configure_geoip( celery_app.app_config.get('geoip', 'db_path'), raven_client=raven_client, _client=_geoip_db) # configure data queues celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES]) celery_app.data_queues = data_queues = configure_data(redis_client) all_queues = all_queues.union( set([queue.key for queue in data_queues.values() if queue.key]))
def init_worker(celery_app, _db=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): """ Configure the passed in celery app, usually stored in :data:`ichnaea.async.app.celery_app`. Does connection, settings and queue setup. Attaches some additional functionality to the :class:`celery.Celery` instance. This is executed inside each forked worker process. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. """ # configure outside connections celery_app.db = configure_db('rw', _db=_db) celery_app.raven_client = raven_client = configure_raven( transport='threaded', _client=_raven_client) celery_app.redis_client = redis_client = configure_redis( _client=_redis_client) celery_app.stats_client = configure_stats(_client=_stats_client) celery_app.geoip_db = configure_geoip(raven_client=raven_client, _client=_geoip_db) # configure data queues celery_app.all_queues = all_queues = set([q.name for q in TASK_QUEUES]) celery_app.data_queues = data_queues = configure_data(redis_client) all_queues = all_queues.union( set([queue.key for queue in data_queues.values() if queue.key]))
def init_worker(celery_app, app_config, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): # currently db_ro is not set up # make config file settings available celery_app.settings = app_config.asdict() # configure outside connections celery_app.db_rw = configure_db( app_config.get('ichnaea', 'db_master'), _db=_db_rw) celery_app.raven_client = raven_client = configure_raven( app_config.get('ichnaea', 'sentry_dsn'), transport='threaded', _client=_raven_client) celery_app.redis_client = redis_client = configure_redis( app_config.get('ichnaea', 'redis_url'), _client=_redis_client) celery_app.stats_client = configure_stats( app_config.get('ichnaea', 'statsd_host'), _client=_stats_client) celery_app.geoip_db = configure_geoip( app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client, _client=_geoip_db) # configure data / export queues celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES]) celery_app.data_queues = data_queues = configure_data(redis_client) for queue in data_queues.values(): if queue.monitor_name: all_queues.add(queue.monitor_name) celery_app.export_queues = configure_export(redis_client, app_config) for queue in celery_app.export_queues.values(): if queue.monitor_name: all_queues.add(queue.monitor_name)
def _open_db(cls, raven, filename=GEOIP_TEST_FILE, mode=MODE_AUTO): return geoip.configure_geoip( filename, mode=mode, raven_client=raven)
def main( ping_connections=False, _db=None, _geoip_db=None, _http_session=None, _raven_client=None, _redis_client=None, _position_searcher=None, _region_searcher=None, ): """ Configure the web app stored in :data:`ichnaea.webapp.app._APP`. Does connection, logging and view config setup. Attaches some additional functionality to the :class:`pyramid.registry.Registry` instance. At startup ping all outbound connections like the database once, to ensure they are actually up and responding. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param ping_connections: If True, ping and test outside connections. :type ping_connections: bool :returns: A configured WSGI app, the result of calling :meth:`pyramid.config.Configurator.make_wsgi_app`. """ configure_logging() config = Configurator() check_config() # add support for pt templates config.include("pyramid_chameleon") # add a config setting to skip logging for some views config.registry.skip_logging = set() configure_api(config) configure_content(config) configure_monitor(config) # configure outside connections registry = config.registry registry.db = configure_db("ro", _db=_db) registry.raven_client = raven_client = configure_raven( transport="gevent", tags={"app": "webapp"}, _client=_raven_client ) registry.redis_client = redis_client = configure_redis(_client=_redis_client) configure_stats() registry.http_session = configure_http_session(_session=_http_session) registry.geoip_db = geoip_db = configure_geoip( raven_client=raven_client, _client=_geoip_db ) # Needs to be the exact same as the *_incoming entries in taskapp.config. registry.data_queues = data_queues = { "update_incoming": DataQueue( "update_incoming", redis_client, "report", batch=100, compress=True ) } for name, func, default in ( ("position_searcher", configure_position_searcher, _position_searcher), ("region_searcher", configure_region_searcher, _region_searcher), ): searcher = func( geoip_db=geoip_db, raven_client=raven_client, redis_client=redis_client, data_queues=data_queues, _searcher=default, ) setattr(registry, name, searcher) config.add_tween("ichnaea.db.db_tween_factory", under=EXCVIEW) config.add_tween("ichnaea.log.log_tween_factory", under=EXCVIEW) config.add_request_method(db_session, property=True) # freeze skip logging set config.registry.skip_logging = frozenset(config.registry.skip_logging) # Should we try to initialize and establish the outbound connections? if ping_connections: with db_worker_session(registry.db, commit=False) as session: ping_session(session) registry.redis_client.ping() return config.make_wsgi_app()
def main(global_config, app_config, init=False, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): configure_logging() # make config file settings available config = Configurator(settings=app_config.asdict()) # add support for pt templates config.include('pyramid_chameleon') configure_content(config) configure_service(config) # configure outside connections registry = config.registry registry.db_rw = configure_db(app_config.get('ichnaea', 'db_master'), _db=_db_rw) registry.db_ro = configure_db(app_config.get('ichnaea', 'db_slave'), _db=_db_ro) registry.raven_client = raven_client = configure_raven( app_config.get('ichnaea', 'sentry_dsn'), transport='gevent', _client=_raven_client) registry.redis_client = configure_redis(app_config.get( 'ichnaea', 'redis_url'), _client=_redis_client) registry.stats_client = configure_stats(app_config.get( 'ichnaea', 'statsd_host'), _client=_stats_client) registry.geoip_db = configure_geoip(app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client, _client=_geoip_db) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW) config.add_request_method(db_rw_session, property=True) config.add_request_method(db_ro_session, property=True) # replace json renderer with custom json variant config.add_renderer('json', customjson.Renderer()) # Should we try to initialize and establish the outbound connections? if init: # pragma: no cover registry.db_ro.ping() registry.redis_client.ping() registry.stats_client.ping() return config.make_wsgi_app()
def main(global_config, heka_config=None, init=False, _db_master=None, _db_slave=None, _heka_client=None, _redis=None, _stats_client=None, **settings): config = Configurator(settings=settings) # add support for pt templates config.include('pyramid_chameleon') settings = config.registry.settings from ichnaea.content.views import configure_content from ichnaea.logging import configure_heka from ichnaea.logging import configure_stats from ichnaea.service import configure_service configure_content(config) configure_service(config) # configure databases incl. test override hooks if _db_master is None: config.registry.db_master = Database(settings['db_master']) else: config.registry.db_master = _db_master if _db_slave is None: config.registry.db_slave = Database(settings['db_slave']) else: config.registry.db_slave = _db_slave if _redis is None: config.registry.redis_client = None if 'redis_url' in settings: config.registry.redis_client = redis_client(settings['redis_url']) else: config.registry.redis_client = _redis config.registry.geoip_db = configure_geoip(config.registry.settings) if _heka_client is None: config.registry.heka_client = configure_heka(heka_config) else: config.registry.heka_client = _heka_client config.registry.stats_client = configure_stats( settings.get('statsd_host'), _client=_stats_client) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.logging.log_tween_factory', under=EXCVIEW) config.add_request_method(db_master_session, property=True) config.add_request_method(db_slave_session, property=True) # replace json renderer with custom json variant config.add_renderer('json', customjson.Renderer()) # Should we try to initialize and establish the outbound connections? if init: # Test the slave DB connection with db_worker_session(config.registry.db_slave) as session: try: session.execute(select([func.now()])).first() except OperationalError: # Let the instance start, so it can recover / reconnect # to the DB later, but provide degraded service in the # meantime. pass # Test the redis connection try: config.registry.redis_client.ping() except ConnectionError: # Same as for the DB, continue with degraded service. pass return config.make_wsgi_app()
def _open_db(cls, filename=None, mode=MODE_AUTO): if filename is None: filename = GEOIP_TEST_FILE return configure_geoip(filename, mode=mode, raven_client=cls.raven_client)
def _open_db(cls, filename=None, mode=MODE_AUTO): if filename is None: filename = GEOIP_TEST_FILE return configure_geoip( filename, mode=mode, raven_client=cls.raven_client)
def configure_geoip(cls, filename=None, mode=MODE_AUTO, raven_client=None): if filename is None: filename = GEOIP_TEST_FILE return configure_geoip(filename=filename, mode=mode, raven_client=raven_client)
def main(app_config, ping_connections=False, _db_rw=None, _db_ro=None, _geoip_db=None, _http_session=None, _raven_client=None, _redis_client=None, _stats_client=None, _position_searcher=None, _region_searcher=None): """ Configure the web app stored in :data:`ichnaea.webapp.app._APP`. Does connection, logging and view config setup. Attaches some additional functionality to the :class:`pyramid.registry.Registry` instance. At startup ping all outbound connections like the database once, to ensure they are actually up and responding. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param app_config: The parsed application ini. :type app_config: :class:`ichnaea.config.Config` :param ping_connections: If True, ping and test outside connections. :type ping_connections: bool :returns: A configured WSGI app, the result of calling :meth:`pyramid.config.Configurator.make_wsgi_app`. """ configure_logging() # make config file settings available config = Configurator(settings=app_config.asdict()) # add support for pt templates config.include('pyramid_chameleon') # add a config setting to skip logging for some views config.registry.skip_logging = set() configure_api(config) configure_content(config) configure_monitor(config) # configure outside connections registry = config.registry registry.db_rw = configure_db( app_config.get('database', 'rw_url'), _db=_db_rw) registry.db_ro = configure_db( app_config.get('database', 'ro_url'), _db=_db_ro) registry.raven_client = raven_client = configure_raven( app_config.get('sentry', 'dsn'), transport='gevent', _client=_raven_client) registry.redis_client = redis_client = configure_redis( app_config.get('cache', 'cache_url'), _client=_redis_client) registry.stats_client = stats_client = configure_stats( app_config, _client=_stats_client) registry.http_session = configure_http_session(_session=_http_session) registry.geoip_db = geoip_db = configure_geoip( app_config.get('geoip', 'db_path'), raven_client=raven_client, _client=_geoip_db) for name, func, default in (('position_searcher', configure_position_searcher, _position_searcher), ('region_searcher', configure_region_searcher, _region_searcher)): searcher = func(app_config, geoip_db=geoip_db, raven_client=raven_client, redis_client=redis_client, stats_client=stats_client, _searcher=default) setattr(registry, name, searcher) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW) config.add_request_method(db_ro_session, property=True) # Add special JSON renderer with nicer float representation config.add_renderer('floatjson', floatjson.FloatJSONRenderer()) # Add text-as-JS renderer. config.add_renderer('js', renderers.JSRenderer()) # freeze skip logging set config.registry.skip_logging = frozenset(config.registry.skip_logging) # Should we try to initialize and establish the outbound connections? if ping_connections: # pragma: no cover registry.db_ro.ping() registry.redis_client.ping() return config.make_wsgi_app()
def geoip_db(raven_client): geoip_db = configure_geoip( GEOIP_TEST_FILE, mode=MODE_AUTO, raven_client=raven_client) yield geoip_db geoip_db.close()
def test_open_ok(self): result = geoip.configure_geoip(filename=self.filename) self.assertIsInstance(result, geoip.GeoIPWrapper)
def main(global_config, heka_config=None, init=False, _db_master=None, _db_slave=None, _heka_client=None, _redis=None, _stats_client=None, **settings): config = Configurator(settings=settings) # add support for pt templates config.include('pyramid_chameleon') settings = config.registry.settings from ichnaea.content.views import configure_content from ichnaea.logging import configure_heka from ichnaea.logging import configure_stats from ichnaea.service import configure_service configure_content(config) configure_service(config) # configure databases incl. test override hooks if _db_master is None: config.registry.db_master = Database(settings['db_master']) else: config.registry.db_master = _db_master if _db_slave is None: config.registry.db_slave = Database(settings['db_slave']) else: config.registry.db_slave = _db_slave if _redis is None: config.registry.redis_client = None if 'redis_url' in settings: config.registry.redis_client = redis_client(settings['redis_url']) else: config.registry.redis_client = _redis if _heka_client is None: # pragma: no cover config.registry.heka_client = heka_client = configure_heka(heka_config) else: config.registry.heka_client = heka_client = _heka_client config.registry.stats_client = configure_stats(settings.get('statsd_host'), _client=_stats_client) config.registry.geoip_db = configure_geoip(config.registry.settings, heka_client=heka_client) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.logging.log_tween_factory', under=EXCVIEW) config.add_request_method(db_master_session, property=True) config.add_request_method(db_slave_session, property=True) # replace json renderer with custom json variant config.add_renderer('json', customjson.Renderer()) # Should we try to initialize and establish the outbound connections? if init: # pragma: no cover # Test the slave DB connection with db_worker_session(config.registry.db_slave) as session: try: session.execute(select([func.now()])).first() except OperationalError: # Let the instance start, so it can recover / reconnect # to the DB later, but provide degraded service in the # meantime. pass # Test the redis connection try: config.registry.redis_client.ping() except ConnectionError: # Same as for the DB, continue with degraded service. pass return config.make_wsgi_app()
def _open_db(self, path=None): if path is None: path = self.filename return geoip.configure_geoip(filename=path)
def geoip_db(raven_client): geoip_db = configure_geoip(mode=MODE_AUTO, raven_client=raven_client) yield geoip_db geoip_db.close()
def main(global_config, heka_config=None, init=False, _db_master=None, _db_slave=None, _heka_client=None, _redis=None, _stats_client=None, **settings): config = Configurator(settings=settings) # add support for pt templates config.include('pyramid_chameleon') settings = config.registry.settings from ichnaea.content.views import configure_content from ichnaea.logging import configure_heka from ichnaea.logging import configure_stats from ichnaea.service import configure_service configure_content(config) configure_service(config) # configure databases incl. test override hooks if _db_master is None: config.registry.db_master = Database(settings['db_master']) else: config.registry.db_master = _db_master if _db_slave is None: config.registry.db_slave = Database(settings['db_slave']) else: config.registry.db_slave = _db_slave if _redis is None: config.registry.redis_client = None if 'redis_url' in settings: config.registry.redis_client = redis_client(settings['redis_url']) else: config.registry.redis_client = _redis if _heka_client is None: # pragma: no cover config.registry.heka_client = heka_client = configure_heka(heka_config) else: config.registry.heka_client = heka_client = _heka_client config.registry.stats_client = configure_stats(settings.get('statsd_host'), _client=_stats_client) config.registry.geoip_db = configure_geoip(config.registry.settings, heka_client=heka_client) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.logging.log_tween_factory', under=EXCVIEW) config.add_request_method(db_master_session, property=True) config.add_request_method(db_slave_session, property=True) # replace json renderer with custom json variant config.add_renderer('json', customjson.Renderer()) # Should we try to initialize and establish the outbound connections? if init: # pragma: no cover registry = config.registry registry.db_slave.ping() registry.redis_client.ping() registry.stats_client.ping() return config.make_wsgi_app()
def main(ping_connections=False, _db=None, _geoip_db=None, _http_session=None, _raven_client=None, _redis_client=None, _stats_client=None, _position_searcher=None, _region_searcher=None): """ Configure the web app stored in :data:`ichnaea.webapp.app._APP`. Does connection, logging and view config setup. Attaches some additional functionality to the :class:`pyramid.registry.Registry` instance. At startup ping all outbound connections like the database once, to ensure they are actually up and responding. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param ping_connections: If True, ping and test outside connections. :type ping_connections: bool :returns: A configured WSGI app, the result of calling :meth:`pyramid.config.Configurator.make_wsgi_app`. """ configure_logging() config = Configurator() # add support for pt templates config.include('pyramid_chameleon') # add a config setting to skip logging for some views config.registry.skip_logging = set() configure_api(config) configure_content(config) configure_monitor(config) # configure outside connections registry = config.registry registry.db = configure_db('ro', _db=_db) registry.raven_client = raven_client = configure_raven( transport='gevent', _client=_raven_client) registry.redis_client = redis_client = configure_redis( _client=_redis_client) registry.stats_client = stats_client = configure_stats( _client=_stats_client) registry.http_session = configure_http_session(_session=_http_session) registry.geoip_db = geoip_db = configure_geoip( raven_client=raven_client, _client=_geoip_db) # Needs to be the exact same as the *_incoming entries in async.config. registry.data_queues = data_queues = { 'update_incoming': DataQueue('update_incoming', redis_client, batch=100, compress=True), } for name, func, default in (('position_searcher', configure_position_searcher, _position_searcher), ('region_searcher', configure_region_searcher, _region_searcher)): searcher = func(geoip_db=geoip_db, raven_client=raven_client, redis_client=redis_client, stats_client=stats_client, data_queues=data_queues, _searcher=default) setattr(registry, name, searcher) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW) config.add_request_method(db_session, property=True) # freeze skip logging set config.registry.skip_logging = frozenset(config.registry.skip_logging) # Should we try to initialize and establish the outbound connections? if ping_connections: # pragma: no cover registry.db.ping() registry.redis_client.ping() return config.make_wsgi_app()
def _open_db(cls, raven, filename=GEOIP_TEST_FILE, mode=MODE_AUTO): return geoip.configure_geoip(filename, mode=mode, raven_client=raven)
def setup_geoip(cls): filename = os.path.join(os.path.dirname(__file__), 'GeoIPCity.dat') cls.geoip_db = configure_geoip(filename=filename)
def main(app_config, ping_connections=False, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): """ Configure the web app stored in :data:`ichnaea.webapp.app._APP`. Does connection, logging and view config setup. Attaches some additional functionality to the :class:`pyramid.registry.Registry` instance. At startup ping all outbound connections like the database once, to ensure they are actually up and responding. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param app_config: The parsed application ini. :type app_config: :class:`ichnaea.config.Config` :param ping_connections: If True, ping and test outside connections. :type ping_connections: bool :returns: A configured WSGI app, the result of calling :meth:`pyramid.config.Configurator.make_wsgi_app`. """ configure_logging() # make config file settings available config = Configurator(settings=app_config.asdict()) # add support for pt templates config.include('pyramid_chameleon') configure_api(config) configure_content(config) configure_monitor(config) # configure outside connections registry = config.registry registry.db_rw = configure_db( app_config.get('ichnaea', 'db_master'), _db=_db_rw) registry.db_ro = configure_db( app_config.get('ichnaea', 'db_slave'), _db=_db_ro) registry.raven_client = raven_client = configure_raven( app_config.get('ichnaea', 'sentry_dsn'), transport='gevent', _client=_raven_client) registry.redis_client = configure_redis( app_config.get('ichnaea', 'redis_url'), _client=_redis_client) registry.stats_client = configure_stats( app_config.get('ichnaea', 'statsd_host'), _client=_stats_client) registry.geoip_db = configure_geoip( app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client, _client=_geoip_db) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW) config.add_request_method(db_rw_session, property=True) config.add_request_method(db_ro_session, property=True) # replace json renderer with custom json variant config.add_renderer('json', customjson.Renderer()) # Should we try to initialize and establish the outbound connections? if ping_connections: # pragma: no cover registry.db_ro.ping() registry.redis_client.ping() registry.stats_client.ping() return config.make_wsgi_app()
def main(app_config, ping_connections=False, _db_rw=None, _db_ro=None, _geoip_db=None, _http_session=None, _raven_client=None, _redis_client=None, _stats_client=None, _country_searcher=None, _position_searcher=None): """ Configure the web app stored in :data:`ichnaea.webapp.app._APP`. Does connection, logging and view config setup. Attaches some additional functionality to the :class:`pyramid.registry.Registry` instance. At startup ping all outbound connections like the database once, to ensure they are actually up and responding. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param app_config: The parsed application ini. :type app_config: :class:`ichnaea.config.Config` :param ping_connections: If True, ping and test outside connections. :type ping_connections: bool :returns: A configured WSGI app, the result of calling :meth:`pyramid.config.Configurator.make_wsgi_app`. """ configure_logging() # make config file settings available config = Configurator(settings=app_config.asdict()) # add support for pt templates config.include('pyramid_chameleon') # add a config setting to skip logging for some views config.registry.skip_logging = set() configure_api(config) configure_content(config) configure_monitor(config) # configure outside connections registry = config.registry registry.db_rw = configure_db( app_config.get('database', 'rw_url'), _db=_db_rw) registry.db_ro = configure_db( app_config.get('database', 'ro_url'), _db=_db_ro) registry.raven_client = raven_client = configure_raven( app_config.get('sentry', 'dsn'), transport='gevent', _client=_raven_client) registry.redis_client = redis_client = configure_redis( app_config.get('cache', 'cache_url'), _client=_redis_client) registry.stats_client = stats_client = configure_stats( app_config, _client=_stats_client) registry.http_session = configure_http_session(_session=_http_session) registry.geoip_db = geoip_db = configure_geoip( app_config.get('geoip', 'db_path'), raven_client=raven_client, _client=_geoip_db) for name, func, default in (('country_searcher', configure_country_searcher, _country_searcher), ('position_searcher', configure_position_searcher, _position_searcher)): searcher = func(app_config, geoip_db=geoip_db, raven_client=raven_client, redis_client=redis_client, stats_client=stats_client, _searcher=default) setattr(registry, name, searcher) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW) config.add_request_method(db_rw_session, property=True) config.add_request_method(db_ro_session, property=True) # Add special JSON renderer with nicer float representation config.add_renderer('floatjson', floatjson.FloatJSONRenderer()) # freeze skip logging set config.registry.skip_logging = frozenset(config.registry.skip_logging) # Should we try to initialize and establish the outbound connections? if ping_connections: # pragma: no cover registry.db_ro.ping() registry.redis_client.ping() return config.make_wsgi_app()