Esempio n. 1
0
    def test_open_fail(self):
        # FIXME going to fail at this point due to GeoIPNull
        with self.assertRaises(geoip.GeoIPError):
            geoip.configure_geoip(filename='/i/taught/i/taw/a/putty/tat')

        with tempfile.NamedTemporaryFile() as temp:
            temp.write('Bucephalus')
            temp.seek(0)
            with self.assertRaises(geoip.GeoIPError):
                geoip.configure_geoip(filename=temp.name)
Esempio n. 2
0
    def test_open_fail(self):
        # FIXME going to fail at this point due to GeoIPNull
        with self.assertRaises(geoip.GeoIPError):
            geoip.configure_geoip(filename='/i/taught/i/taw/a/putty/tat')

        with tempfile.NamedTemporaryFile() as temp:
            temp.write('Bucephalus')
            temp.seek(0)
            with self.assertRaises(geoip.GeoIPError):
                geoip.configure_geoip(filename=temp.name)
Esempio n. 3
0
def init_worker(
    celery_app,
    app_config,
    _db_rw=None,
    _db_ro=None,
    _geoip_db=None,
    _raven_client=None,
    _redis_client=None,
    _stats_client=None,
):
    """
    Configure the passed in celery app, usually stored in
    :data:`ichnaea.async.app.celery_app`.

    Does connection, settings and queue setup. Attaches some
    additional functionality to the :class:`celery.Celery` instance.

    This is executed inside each forked worker process.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param _db_ro: Ignored, read-only database connection isn't used.
    """

    # make config file settings available
    celery_app.settings = app_config.asdict()

    # configure outside connections
    celery_app.db_rw = configure_db(app_config.get("ichnaea", "db_master"), _db=_db_rw)

    celery_app.raven_client = raven_client = configure_raven(
        app_config.get("ichnaea", "sentry_dsn"), transport="threaded", _client=_raven_client
    )

    celery_app.redis_client = redis_client = configure_redis(
        app_config.get("ichnaea", "redis_url"), _client=_redis_client
    )

    celery_app.stats_client = configure_stats(app_config.get("ichnaea", "statsd_host"), _client=_stats_client)

    celery_app.geoip_db = configure_geoip(
        app_config.get("ichnaea", "geoip_db_path"), raven_client=raven_client, _client=_geoip_db
    )

    # configure data / export queues
    celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES])

    celery_app.data_queues = data_queues = configure_data(redis_client)
    for queue in data_queues.values():
        if queue.monitor_name:
            all_queues.add(queue.monitor_name)

    celery_app.export_queues = configure_export(redis_client, app_config)
    for queue in celery_app.export_queues.values():
        if queue.monitor_name:
            all_queues.add(queue.monitor_name)
Esempio n. 4
0
def init_worker(celery_app,
                app_config,
                _db_rw=None,
                _db_ro=None,
                _geoip_db=None,
                _raven_client=None,
                _redis_client=None,
                _stats_client=None):
    """
    Configure the passed in celery app, usually stored in
    :data:`ichnaea.async.app.celery_app`.

    Does connection, settings and queue setup. Attaches some
    additional functionality to the :class:`celery.Celery` instance.

    This is executed inside each forked worker process.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param _db_ro: Ignored, read-only database connection isn't used.
    """

    # make config file settings available
    celery_app.settings = app_config.asdict()

    # configure outside connections
    celery_app.db_rw = configure_db(app_config.get('database', 'rw_url'),
                                    _db=_db_rw)

    celery_app.raven_client = raven_client = configure_raven(
        app_config.get('sentry', 'dsn'),
        transport='threaded',
        _client=_raven_client)

    celery_app.redis_client = redis_client = configure_redis(
        app_config.get('cache', 'cache_url'), _client=_redis_client)

    celery_app.stats_client = configure_stats(app_config,
                                              _client=_stats_client)

    celery_app.geoip_db = configure_geoip(app_config.get('geoip', 'db_path'),
                                          raven_client=raven_client,
                                          _client=_geoip_db)

    # configure data / export queues
    celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES])

    celery_app.data_queues = data_queues = configure_data(redis_client)
    for queue in data_queues.values():
        if queue.monitor_name:
            all_queues.add(queue.monitor_name)

    celery_app.export_queues = configure_export(redis_client, app_config)
    for queue in celery_app.export_queues.values():
        if queue.monitor_name:
            all_queues.add(queue.monitor_name)
Esempio n. 5
0
def main(global_config, app_config, init=False,
         _db_rw=None, _db_ro=None, _geoip_db=None,
         _raven_client=None, _redis_client=None, _stats_client=None):

    configure_logging()

    # make config file settings available
    config = Configurator(settings=app_config.asdict())

    # add support for pt templates
    config.include('pyramid_chameleon')

    configure_content(config)
    configure_service(config)

    # configure outside connections
    registry = config.registry

    registry.db_rw = configure_db(
        app_config.get('ichnaea', 'db_master'), _db=_db_rw)
    registry.db_ro = configure_db(
        app_config.get('ichnaea', 'db_slave'), _db=_db_ro)

    registry.raven_client = raven_client = configure_raven(
        app_config.get('ichnaea', 'sentry_dsn'),
        transport='gevent', _client=_raven_client)

    registry.redis_client = configure_redis(
        app_config.get('ichnaea', 'redis_url'), _client=_redis_client)

    registry.stats_client = configure_stats(
        app_config.get('ichnaea', 'statsd_host'), _client=_stats_client)

    registry.geoip_db = configure_geoip(
        app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client,
        _client=_geoip_db)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_rw_session, property=True)
    config.add_request_method(db_ro_session, property=True)

    # replace json renderer with custom json variant
    config.add_renderer('json', customjson.Renderer())

    # Should we try to initialize and establish the outbound connections?
    if init:  # pragma: no cover
        registry.db_ro.ping()
        registry.redis_client.ping()
        registry.stats_client.ping()

    return config.make_wsgi_app()
Esempio n. 6
0
def main(global_config, _db_master=None, _db_slave=None, **settings):
    config = Configurator(settings=settings)

    # add support for pt templates
    config.include('pyramid_chameleon')

    settings = config.registry.settings

    from ichnaea.content.views import configure_content
    from ichnaea.service import configure_service
    from ichnaea.heka_logging import configure_heka

    configure_content(config)
    configure_service(config)

    # configure databases incl. test override hooks
    if _db_master is None:
        config.registry.db_master = Database(
            settings['db_master'],
            socket=settings.get('db_master_socket'),
        )
    else:
        config.registry.db_master = _db_master
    if _db_slave is None:
        config.registry.db_slave = Database(
            settings['db_slave'],
            socket=settings.get('db_slave_socket'),
            create=False,
        )
    else:
        config.registry.db_slave = _db_slave

    config.registry.geoip_db = configure_geoip(config.registry.settings)

    config.registry.heka_client = configure_heka(config.registry.settings)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.heka_logging.heka_tween_factory', under=EXCVIEW)
    config.add_request_method(db_master_session, property=True)
    config.add_request_method(db_slave_session, property=True)

    # replace json renderer with decimal json variant
    config.add_renderer('json', decimaljson.Renderer())
    return config.make_wsgi_app()
Esempio n. 7
0
def init_worker(celery_app,
                _db_rw=None, _db_ro=None, _geoip_db=None,
                _raven_client=None, _redis_client=None, _stats_client=None):
    """
    Configure the passed in celery app, usually stored in
    :data:`ichnaea.async.app.celery_app`.

    Does connection, settings and queue setup. Attaches some
    additional functionality to the :class:`celery.Celery` instance.

    This is executed inside each forked worker process.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param _db_ro: Ignored, read-only database connection isn't used.
    """

    # configure outside connections
    celery_app.db_rw = configure_db(
        celery_app.app_config.get('database', 'rw_url'), _db=_db_rw)

    celery_app.raven_client = raven_client = configure_raven(
        celery_app.app_config.get('sentry', 'dsn'),
        transport='threaded', _client=_raven_client)

    celery_app.redis_client = redis_client = configure_redis(
        celery_app.app_config.get('cache', 'cache_url'), _client=_redis_client)

    celery_app.stats_client = configure_stats(
        celery_app.app_config, _client=_stats_client)

    celery_app.geoip_db = configure_geoip(
        celery_app.app_config.get('geoip', 'db_path'),
        raven_client=raven_client, _client=_geoip_db)

    # configure data queues
    celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES])

    celery_app.data_queues = data_queues = configure_data(redis_client)
    all_queues = all_queues.union(
        set([queue.key for queue in data_queues.values() if queue.key]))
def init_worker(celery_app,
                _db=None,
                _geoip_db=None,
                _raven_client=None,
                _redis_client=None,
                _stats_client=None):
    """
    Configure the passed in celery app, usually stored in
    :data:`ichnaea.async.app.celery_app`.

    Does connection, settings and queue setup. Attaches some
    additional functionality to the :class:`celery.Celery` instance.

    This is executed inside each forked worker process.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.
    """

    # configure outside connections
    celery_app.db = configure_db('rw', _db=_db)

    celery_app.raven_client = raven_client = configure_raven(
        transport='threaded', _client=_raven_client)

    celery_app.redis_client = redis_client = configure_redis(
        _client=_redis_client)

    celery_app.stats_client = configure_stats(_client=_stats_client)

    celery_app.geoip_db = configure_geoip(raven_client=raven_client,
                                          _client=_geoip_db)

    # configure data queues
    celery_app.all_queues = all_queues = set([q.name for q in TASK_QUEUES])

    celery_app.data_queues = data_queues = configure_data(redis_client)
    all_queues = all_queues.union(
        set([queue.key for queue in data_queues.values() if queue.key]))
Esempio n. 9
0
def init_worker(celery_app, app_config,
                _db_rw=None, _db_ro=None, _geoip_db=None,
                _raven_client=None, _redis_client=None, _stats_client=None):
    # currently db_ro is not set up

    # make config file settings available
    celery_app.settings = app_config.asdict()

    # configure outside connections
    celery_app.db_rw = configure_db(
        app_config.get('ichnaea', 'db_master'), _db=_db_rw)

    celery_app.raven_client = raven_client = configure_raven(
        app_config.get('ichnaea', 'sentry_dsn'),
        transport='threaded', _client=_raven_client)

    celery_app.redis_client = redis_client = configure_redis(
        app_config.get('ichnaea', 'redis_url'), _client=_redis_client)

    celery_app.stats_client = configure_stats(
        app_config.get('ichnaea', 'statsd_host'), _client=_stats_client)

    celery_app.geoip_db = configure_geoip(
        app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client,
        _client=_geoip_db)

    # configure data / export queues
    celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES])

    celery_app.data_queues = data_queues = configure_data(redis_client)
    for queue in data_queues.values():
        if queue.monitor_name:
            all_queues.add(queue.monitor_name)

    celery_app.export_queues = configure_export(redis_client, app_config)
    for queue in celery_app.export_queues.values():
        if queue.monitor_name:
            all_queues.add(queue.monitor_name)
Esempio n. 10
0
 def _open_db(cls, raven, filename=GEOIP_TEST_FILE, mode=MODE_AUTO):
     return geoip.configure_geoip(
         filename, mode=mode, raven_client=raven)
Esempio n. 11
0
def main(
    ping_connections=False,
    _db=None,
    _geoip_db=None,
    _http_session=None,
    _raven_client=None,
    _redis_client=None,
    _position_searcher=None,
    _region_searcher=None,
):
    """
    Configure the web app stored in :data:`ichnaea.webapp.app._APP`.

    Does connection, logging and view config setup. Attaches some
    additional functionality to the :class:`pyramid.registry.Registry`
    instance.

    At startup ping all outbound connections like the database
    once, to ensure they are actually up and responding.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param ping_connections: If True, ping and test outside connections.
    :type ping_connections: bool

    :returns: A configured WSGI app, the result of calling
              :meth:`pyramid.config.Configurator.make_wsgi_app`.
    """

    configure_logging()

    config = Configurator()
    check_config()

    # add support for pt templates
    config.include("pyramid_chameleon")

    # add a config setting to skip logging for some views
    config.registry.skip_logging = set()

    configure_api(config)
    configure_content(config)
    configure_monitor(config)

    # configure outside connections
    registry = config.registry

    registry.db = configure_db("ro", _db=_db)

    registry.raven_client = raven_client = configure_raven(
        transport="gevent", tags={"app": "webapp"}, _client=_raven_client
    )

    registry.redis_client = redis_client = configure_redis(_client=_redis_client)

    configure_stats()

    registry.http_session = configure_http_session(_session=_http_session)

    registry.geoip_db = geoip_db = configure_geoip(
        raven_client=raven_client, _client=_geoip_db
    )

    # Needs to be the exact same as the *_incoming entries in taskapp.config.
    registry.data_queues = data_queues = {
        "update_incoming": DataQueue(
            "update_incoming", redis_client, "report", batch=100, compress=True
        )
    }

    for name, func, default in (
        ("position_searcher", configure_position_searcher, _position_searcher),
        ("region_searcher", configure_region_searcher, _region_searcher),
    ):
        searcher = func(
            geoip_db=geoip_db,
            raven_client=raven_client,
            redis_client=redis_client,
            data_queues=data_queues,
            _searcher=default,
        )
        setattr(registry, name, searcher)

    config.add_tween("ichnaea.db.db_tween_factory", under=EXCVIEW)
    config.add_tween("ichnaea.log.log_tween_factory", under=EXCVIEW)
    config.add_request_method(db_session, property=True)

    # freeze skip logging set
    config.registry.skip_logging = frozenset(config.registry.skip_logging)

    # Should we try to initialize and establish the outbound connections?
    if ping_connections:
        with db_worker_session(registry.db, commit=False) as session:
            ping_session(session)
        registry.redis_client.ping()

    return config.make_wsgi_app()
Esempio n. 12
0
def main(global_config,
         app_config,
         init=False,
         _db_rw=None,
         _db_ro=None,
         _geoip_db=None,
         _raven_client=None,
         _redis_client=None,
         _stats_client=None):

    configure_logging()

    # make config file settings available
    config = Configurator(settings=app_config.asdict())

    # add support for pt templates
    config.include('pyramid_chameleon')

    configure_content(config)
    configure_service(config)

    # configure outside connections
    registry = config.registry

    registry.db_rw = configure_db(app_config.get('ichnaea', 'db_master'),
                                  _db=_db_rw)
    registry.db_ro = configure_db(app_config.get('ichnaea', 'db_slave'),
                                  _db=_db_ro)

    registry.raven_client = raven_client = configure_raven(
        app_config.get('ichnaea', 'sentry_dsn'),
        transport='gevent',
        _client=_raven_client)

    registry.redis_client = configure_redis(app_config.get(
        'ichnaea', 'redis_url'),
                                            _client=_redis_client)

    registry.stats_client = configure_stats(app_config.get(
        'ichnaea', 'statsd_host'),
                                            _client=_stats_client)

    registry.geoip_db = configure_geoip(app_config.get('ichnaea',
                                                       'geoip_db_path'),
                                        raven_client=raven_client,
                                        _client=_geoip_db)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_rw_session, property=True)
    config.add_request_method(db_ro_session, property=True)

    # replace json renderer with custom json variant
    config.add_renderer('json', customjson.Renderer())

    # Should we try to initialize and establish the outbound connections?
    if init:  # pragma: no cover
        registry.db_ro.ping()
        registry.redis_client.ping()
        registry.stats_client.ping()

    return config.make_wsgi_app()
Esempio n. 13
0
def main(global_config, heka_config=None, init=False,
         _db_master=None, _db_slave=None, _heka_client=None, _redis=None,
         _stats_client=None, **settings):
    config = Configurator(settings=settings)

    # add support for pt templates
    config.include('pyramid_chameleon')

    settings = config.registry.settings

    from ichnaea.content.views import configure_content
    from ichnaea.logging import configure_heka
    from ichnaea.logging import configure_stats
    from ichnaea.service import configure_service

    configure_content(config)
    configure_service(config)

    # configure databases incl. test override hooks
    if _db_master is None:
        config.registry.db_master = Database(settings['db_master'])
    else:
        config.registry.db_master = _db_master
    if _db_slave is None:
        config.registry.db_slave = Database(settings['db_slave'])
    else:
        config.registry.db_slave = _db_slave

    if _redis is None:
        config.registry.redis_client = None
        if 'redis_url' in settings:
            config.registry.redis_client = redis_client(settings['redis_url'])
    else:
        config.registry.redis_client = _redis

    config.registry.geoip_db = configure_geoip(config.registry.settings)

    if _heka_client is None:
        config.registry.heka_client = configure_heka(heka_config)
    else:
        config.registry.heka_client = _heka_client

    config.registry.stats_client = configure_stats(
        settings.get('statsd_host'), _client=_stats_client)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.logging.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_master_session, property=True)
    config.add_request_method(db_slave_session, property=True)

    # replace json renderer with custom json variant
    config.add_renderer('json', customjson.Renderer())

    # Should we try to initialize and establish the outbound connections?
    if init:
        # Test the slave DB connection
        with db_worker_session(config.registry.db_slave) as session:
            try:
                session.execute(select([func.now()])).first()
            except OperationalError:
                # Let the instance start, so it can recover / reconnect
                # to the DB later, but provide degraded service in the
                # meantime.
                pass

        # Test the redis connection
        try:
            config.registry.redis_client.ping()
        except ConnectionError:
            # Same as for the DB, continue with degraded service.
            pass

    return config.make_wsgi_app()
Esempio n. 14
0
 def _open_db(cls, filename=None, mode=MODE_AUTO):
     if filename is None:
         filename = GEOIP_TEST_FILE
     return configure_geoip(filename,
                            mode=mode,
                            raven_client=cls.raven_client)
Esempio n. 15
0
 def _open_db(cls, filename=None, mode=MODE_AUTO):
     if filename is None:
         filename = GEOIP_TEST_FILE
     return configure_geoip(
         filename, mode=mode, raven_client=cls.raven_client)
Esempio n. 16
0
 def configure_geoip(cls, filename=None, mode=MODE_AUTO, raven_client=None):
     if filename is None:
         filename = GEOIP_TEST_FILE
     return configure_geoip(filename=filename,
                            mode=mode,
                            raven_client=raven_client)
Esempio n. 17
0
def main(app_config, ping_connections=False,
         _db_rw=None, _db_ro=None, _geoip_db=None, _http_session=None,
         _raven_client=None, _redis_client=None, _stats_client=None,
         _position_searcher=None, _region_searcher=None):
    """
    Configure the web app stored in :data:`ichnaea.webapp.app._APP`.

    Does connection, logging and view config setup. Attaches some
    additional functionality to the :class:`pyramid.registry.Registry`
    instance.

    At startup ping all outbound connections like the database
    once, to ensure they are actually up and responding.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param app_config: The parsed application ini.
    :type app_config: :class:`ichnaea.config.Config`

    :param ping_connections: If True, ping and test outside connections.
    :type ping_connections: bool

    :returns: A configured WSGI app, the result of calling
              :meth:`pyramid.config.Configurator.make_wsgi_app`.
    """

    configure_logging()

    # make config file settings available
    config = Configurator(settings=app_config.asdict())

    # add support for pt templates
    config.include('pyramid_chameleon')

    # add a config setting to skip logging for some views
    config.registry.skip_logging = set()

    configure_api(config)
    configure_content(config)
    configure_monitor(config)

    # configure outside connections
    registry = config.registry

    registry.db_rw = configure_db(
        app_config.get('database', 'rw_url'), _db=_db_rw)
    registry.db_ro = configure_db(
        app_config.get('database', 'ro_url'), _db=_db_ro)

    registry.raven_client = raven_client = configure_raven(
        app_config.get('sentry', 'dsn'),
        transport='gevent', _client=_raven_client)

    registry.redis_client = redis_client = configure_redis(
        app_config.get('cache', 'cache_url'), _client=_redis_client)

    registry.stats_client = stats_client = configure_stats(
        app_config, _client=_stats_client)

    registry.http_session = configure_http_session(_session=_http_session)

    registry.geoip_db = geoip_db = configure_geoip(
        app_config.get('geoip', 'db_path'), raven_client=raven_client,
        _client=_geoip_db)

    for name, func, default in (('position_searcher',
                                 configure_position_searcher,
                                 _position_searcher),
                                ('region_searcher',
                                 configure_region_searcher,
                                 _region_searcher)):
        searcher = func(app_config,
                        geoip_db=geoip_db, raven_client=raven_client,
                        redis_client=redis_client, stats_client=stats_client,
                        _searcher=default)
        setattr(registry, name, searcher)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_ro_session, property=True)

    # Add special JSON renderer with nicer float representation
    config.add_renderer('floatjson', floatjson.FloatJSONRenderer())

    # Add text-as-JS renderer.
    config.add_renderer('js', renderers.JSRenderer())

    # freeze skip logging set
    config.registry.skip_logging = frozenset(config.registry.skip_logging)

    # Should we try to initialize and establish the outbound connections?
    if ping_connections:  # pragma: no cover
        registry.db_ro.ping()
        registry.redis_client.ping()

    return config.make_wsgi_app()
Esempio n. 18
0
def geoip_db(raven_client):
    geoip_db = configure_geoip(
        GEOIP_TEST_FILE, mode=MODE_AUTO, raven_client=raven_client)
    yield geoip_db
    geoip_db.close()
Esempio n. 19
0
 def test_open_ok(self):
     result = geoip.configure_geoip(filename=self.filename)
     self.assertIsInstance(result, geoip.GeoIPWrapper)
Esempio n. 20
0
def main(global_config,
         heka_config=None,
         init=False,
         _db_master=None,
         _db_slave=None,
         _heka_client=None,
         _redis=None,
         _stats_client=None,
         **settings):
    config = Configurator(settings=settings)

    # add support for pt templates
    config.include('pyramid_chameleon')

    settings = config.registry.settings

    from ichnaea.content.views import configure_content
    from ichnaea.logging import configure_heka
    from ichnaea.logging import configure_stats
    from ichnaea.service import configure_service

    configure_content(config)
    configure_service(config)

    # configure databases incl. test override hooks
    if _db_master is None:
        config.registry.db_master = Database(settings['db_master'])
    else:
        config.registry.db_master = _db_master
    if _db_slave is None:
        config.registry.db_slave = Database(settings['db_slave'])
    else:
        config.registry.db_slave = _db_slave

    if _redis is None:
        config.registry.redis_client = None
        if 'redis_url' in settings:
            config.registry.redis_client = redis_client(settings['redis_url'])
    else:
        config.registry.redis_client = _redis

    if _heka_client is None:  # pragma: no cover
        config.registry.heka_client = heka_client = configure_heka(heka_config)
    else:
        config.registry.heka_client = heka_client = _heka_client

    config.registry.stats_client = configure_stats(settings.get('statsd_host'),
                                                   _client=_stats_client)

    config.registry.geoip_db = configure_geoip(config.registry.settings,
                                               heka_client=heka_client)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.logging.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_master_session, property=True)
    config.add_request_method(db_slave_session, property=True)

    # replace json renderer with custom json variant
    config.add_renderer('json', customjson.Renderer())

    # Should we try to initialize and establish the outbound connections?
    if init:  # pragma: no cover
        # Test the slave DB connection
        with db_worker_session(config.registry.db_slave) as session:
            try:
                session.execute(select([func.now()])).first()
            except OperationalError:
                # Let the instance start, so it can recover / reconnect
                # to the DB later, but provide degraded service in the
                # meantime.
                pass

        # Test the redis connection
        try:
            config.registry.redis_client.ping()
        except ConnectionError:
            # Same as for the DB, continue with degraded service.
            pass

    return config.make_wsgi_app()
Esempio n. 21
0
 def _open_db(self, path=None):
     if path is None:
         path = self.filename
     return geoip.configure_geoip(filename=path)
Esempio n. 22
0
 def _open_db(self, path=None):
     if path is None:
         path = self.filename
     return geoip.configure_geoip(filename=path)
Esempio n. 23
0
def geoip_db(raven_client):
    geoip_db = configure_geoip(mode=MODE_AUTO, raven_client=raven_client)
    yield geoip_db
    geoip_db.close()
Esempio n. 24
0
def geoip_db(raven_client):
    geoip_db = configure_geoip(mode=MODE_AUTO, raven_client=raven_client)
    yield geoip_db
    geoip_db.close()
Esempio n. 25
0
def main(global_config,
         heka_config=None,
         init=False,
         _db_master=None,
         _db_slave=None,
         _heka_client=None,
         _redis=None,
         _stats_client=None,
         **settings):
    config = Configurator(settings=settings)

    # add support for pt templates
    config.include('pyramid_chameleon')

    settings = config.registry.settings

    from ichnaea.content.views import configure_content
    from ichnaea.logging import configure_heka
    from ichnaea.logging import configure_stats
    from ichnaea.service import configure_service

    configure_content(config)
    configure_service(config)

    # configure databases incl. test override hooks
    if _db_master is None:
        config.registry.db_master = Database(settings['db_master'])
    else:
        config.registry.db_master = _db_master
    if _db_slave is None:
        config.registry.db_slave = Database(settings['db_slave'])
    else:
        config.registry.db_slave = _db_slave

    if _redis is None:
        config.registry.redis_client = None
        if 'redis_url' in settings:
            config.registry.redis_client = redis_client(settings['redis_url'])
    else:
        config.registry.redis_client = _redis

    if _heka_client is None:  # pragma: no cover
        config.registry.heka_client = heka_client = configure_heka(heka_config)
    else:
        config.registry.heka_client = heka_client = _heka_client

    config.registry.stats_client = configure_stats(settings.get('statsd_host'),
                                                   _client=_stats_client)

    config.registry.geoip_db = configure_geoip(config.registry.settings,
                                               heka_client=heka_client)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.logging.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_master_session, property=True)
    config.add_request_method(db_slave_session, property=True)

    # replace json renderer with custom json variant
    config.add_renderer('json', customjson.Renderer())

    # Should we try to initialize and establish the outbound connections?
    if init:  # pragma: no cover
        registry = config.registry
        registry.db_slave.ping()
        registry.redis_client.ping()
        registry.stats_client.ping()

    return config.make_wsgi_app()
Esempio n. 26
0
def main(ping_connections=False,
         _db=None, _geoip_db=None, _http_session=None,
         _raven_client=None, _redis_client=None, _stats_client=None,
         _position_searcher=None, _region_searcher=None):
    """
    Configure the web app stored in :data:`ichnaea.webapp.app._APP`.

    Does connection, logging and view config setup. Attaches some
    additional functionality to the :class:`pyramid.registry.Registry`
    instance.

    At startup ping all outbound connections like the database
    once, to ensure they are actually up and responding.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param ping_connections: If True, ping and test outside connections.
    :type ping_connections: bool

    :returns: A configured WSGI app, the result of calling
              :meth:`pyramid.config.Configurator.make_wsgi_app`.
    """

    configure_logging()

    config = Configurator()

    # add support for pt templates
    config.include('pyramid_chameleon')

    # add a config setting to skip logging for some views
    config.registry.skip_logging = set()

    configure_api(config)
    configure_content(config)
    configure_monitor(config)

    # configure outside connections
    registry = config.registry

    registry.db = configure_db('ro', _db=_db)

    registry.raven_client = raven_client = configure_raven(
        transport='gevent', _client=_raven_client)

    registry.redis_client = redis_client = configure_redis(
        _client=_redis_client)

    registry.stats_client = stats_client = configure_stats(
        _client=_stats_client)

    registry.http_session = configure_http_session(_session=_http_session)

    registry.geoip_db = geoip_db = configure_geoip(
        raven_client=raven_client, _client=_geoip_db)

    # Needs to be the exact same as the *_incoming entries in async.config.
    registry.data_queues = data_queues = {
        'update_incoming': DataQueue('update_incoming', redis_client,
                                     batch=100, compress=True),
    }

    for name, func, default in (('position_searcher',
                                 configure_position_searcher,
                                 _position_searcher),
                                ('region_searcher',
                                 configure_region_searcher,
                                 _region_searcher)):
        searcher = func(geoip_db=geoip_db, raven_client=raven_client,
                        redis_client=redis_client, stats_client=stats_client,
                        data_queues=data_queues, _searcher=default)
        setattr(registry, name, searcher)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_session, property=True)

    # freeze skip logging set
    config.registry.skip_logging = frozenset(config.registry.skip_logging)

    # Should we try to initialize and establish the outbound connections?
    if ping_connections:  # pragma: no cover
        registry.db.ping()
        registry.redis_client.ping()

    return config.make_wsgi_app()
Esempio n. 27
0
 def _open_db(cls, raven, filename=GEOIP_TEST_FILE, mode=MODE_AUTO):
     return geoip.configure_geoip(filename, mode=mode, raven_client=raven)
Esempio n. 28
0
 def configure_geoip(cls, filename=None, mode=MODE_AUTO, raven_client=None):
     if filename is None:
         filename = GEOIP_TEST_FILE
     return configure_geoip(filename=filename, mode=mode,
                            raven_client=raven_client)
Esempio n. 29
0
 def setup_geoip(cls):
     filename = os.path.join(os.path.dirname(__file__), 'GeoIPCity.dat')
     cls.geoip_db = configure_geoip(filename=filename)
Esempio n. 30
0
def main(app_config, ping_connections=False,
         _db_rw=None, _db_ro=None, _geoip_db=None,
         _raven_client=None, _redis_client=None, _stats_client=None):
    """
    Configure the web app stored in :data:`ichnaea.webapp.app._APP`.

    Does connection, logging and view config setup. Attaches some
    additional functionality to the :class:`pyramid.registry.Registry`
    instance.

    At startup ping all outbound connections like the database
    once, to ensure they are actually up and responding.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param app_config: The parsed application ini.
    :type app_config: :class:`ichnaea.config.Config`

    :param ping_connections: If True, ping and test outside connections.
    :type ping_connections: bool

    :returns: A configured WSGI app, the result of calling
              :meth:`pyramid.config.Configurator.make_wsgi_app`.
    """

    configure_logging()

    # make config file settings available
    config = Configurator(settings=app_config.asdict())

    # add support for pt templates
    config.include('pyramid_chameleon')

    configure_api(config)
    configure_content(config)
    configure_monitor(config)

    # configure outside connections
    registry = config.registry

    registry.db_rw = configure_db(
        app_config.get('ichnaea', 'db_master'), _db=_db_rw)
    registry.db_ro = configure_db(
        app_config.get('ichnaea', 'db_slave'), _db=_db_ro)

    registry.raven_client = raven_client = configure_raven(
        app_config.get('ichnaea', 'sentry_dsn'),
        transport='gevent', _client=_raven_client)

    registry.redis_client = configure_redis(
        app_config.get('ichnaea', 'redis_url'), _client=_redis_client)

    registry.stats_client = configure_stats(
        app_config.get('ichnaea', 'statsd_host'), _client=_stats_client)

    registry.geoip_db = configure_geoip(
        app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client,
        _client=_geoip_db)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_rw_session, property=True)
    config.add_request_method(db_ro_session, property=True)

    # replace json renderer with custom json variant
    config.add_renderer('json', customjson.Renderer())

    # Should we try to initialize and establish the outbound connections?
    if ping_connections:  # pragma: no cover
        registry.db_ro.ping()
        registry.redis_client.ping()
        registry.stats_client.ping()

    return config.make_wsgi_app()
Esempio n. 31
0
def main(app_config, ping_connections=False,
         _db_rw=None, _db_ro=None, _geoip_db=None, _http_session=None,
         _raven_client=None, _redis_client=None, _stats_client=None,
         _country_searcher=None, _position_searcher=None):
    """
    Configure the web app stored in :data:`ichnaea.webapp.app._APP`.

    Does connection, logging and view config setup. Attaches some
    additional functionality to the :class:`pyramid.registry.Registry`
    instance.

    At startup ping all outbound connections like the database
    once, to ensure they are actually up and responding.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param app_config: The parsed application ini.
    :type app_config: :class:`ichnaea.config.Config`

    :param ping_connections: If True, ping and test outside connections.
    :type ping_connections: bool

    :returns: A configured WSGI app, the result of calling
              :meth:`pyramid.config.Configurator.make_wsgi_app`.
    """

    configure_logging()

    # make config file settings available
    config = Configurator(settings=app_config.asdict())

    # add support for pt templates
    config.include('pyramid_chameleon')

    # add a config setting to skip logging for some views
    config.registry.skip_logging = set()

    configure_api(config)
    configure_content(config)
    configure_monitor(config)

    # configure outside connections
    registry = config.registry

    registry.db_rw = configure_db(
        app_config.get('database', 'rw_url'), _db=_db_rw)
    registry.db_ro = configure_db(
        app_config.get('database', 'ro_url'), _db=_db_ro)

    registry.raven_client = raven_client = configure_raven(
        app_config.get('sentry', 'dsn'),
        transport='gevent', _client=_raven_client)

    registry.redis_client = redis_client = configure_redis(
        app_config.get('cache', 'cache_url'), _client=_redis_client)

    registry.stats_client = stats_client = configure_stats(
        app_config, _client=_stats_client)

    registry.http_session = configure_http_session(_session=_http_session)

    registry.geoip_db = geoip_db = configure_geoip(
        app_config.get('geoip', 'db_path'), raven_client=raven_client,
        _client=_geoip_db)

    for name, func, default in (('country_searcher',
                                 configure_country_searcher,
                                 _country_searcher),
                                ('position_searcher',
                                 configure_position_searcher,
                                 _position_searcher)):
        searcher = func(app_config,
                        geoip_db=geoip_db, raven_client=raven_client,
                        redis_client=redis_client, stats_client=stats_client,
                        _searcher=default)
        setattr(registry, name, searcher)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_rw_session, property=True)
    config.add_request_method(db_ro_session, property=True)

    # Add special JSON renderer with nicer float representation
    config.add_renderer('floatjson', floatjson.FloatJSONRenderer())

    # freeze skip logging set
    config.registry.skip_logging = frozenset(config.registry.skip_logging)

    # Should we try to initialize and establish the outbound connections?
    if ping_connections:  # pragma: no cover
        registry.db_ro.ping()
        registry.redis_client.ping()

    return config.make_wsgi_app()
Esempio n. 32
0
 def test_open_ok(self):
     result = geoip.configure_geoip(filename=self.filename)
     self.assertIsInstance(result, geoip.GeoIPWrapper)