Ejemplo n.º 1
0
def main(global_config, app_config, init=False,
         _db_rw=None, _db_ro=None, _geoip_db=None,
         _raven_client=None, _redis_client=None, _stats_client=None):

    configure_logging()

    # make config file settings available
    config = Configurator(settings=app_config.asdict())

    # add support for pt templates
    config.include('pyramid_chameleon')

    configure_content(config)
    configure_service(config)

    # configure outside connections
    registry = config.registry

    registry.db_rw = configure_db(
        app_config.get('ichnaea', 'db_master'), _db=_db_rw)
    registry.db_ro = configure_db(
        app_config.get('ichnaea', 'db_slave'), _db=_db_ro)

    registry.raven_client = raven_client = configure_raven(
        app_config.get('ichnaea', 'sentry_dsn'),
        transport='gevent', _client=_raven_client)

    registry.redis_client = configure_redis(
        app_config.get('ichnaea', 'redis_url'), _client=_redis_client)

    registry.stats_client = configure_stats(
        app_config.get('ichnaea', 'statsd_host'), _client=_stats_client)

    registry.geoip_db = configure_geoip(
        app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client,
        _client=_geoip_db)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_rw_session, property=True)
    config.add_request_method(db_ro_session, property=True)

    # replace json renderer with custom json variant
    config.add_renderer('json', customjson.Renderer())

    # Should we try to initialize and establish the outbound connections?
    if init:  # pragma: no cover
        registry.db_ro.ping()
        registry.redis_client.ping()
        registry.stats_client.ping()

    return config.make_wsgi_app()
Ejemplo n.º 2
0
def main(argv, _db_rw=None, _redis_client=None):  # pragma: no cover
    parser = argparse.ArgumentParser(prog=argv[0],
                                     description='Load/import cell data.')
    parser.add_argument('--datatype',
                        default='ocid',
                        help='Type of the data file, e.g. ocid')
    parser.add_argument('--filename', help='Path to the gzipped csv file.')

    args = parser.parse_args(argv[1:])
    if not args.filename:
        parser.print_help()
        sys.exit(1)

    filename = os.path.abspath(args.filename)
    if not os.path.isfile(filename):
        print('File not found.')
        sys.exit(1)

    datatype = args.datatype
    if datatype not in ('cell', 'ocid'):
        print('Unknown data type.')
        sys.exit(1)

    configure_logging()
    app_config = read_config()
    db = configure_db(app_config.get('database', 'rw_url'), _db=_db_rw)
    redis_client = configure_redis(app_config.get('cache', 'cache_url'),
                                   _client=_redis_client)

    load_file(db, redis_client, datatype, filename)
def run_migrations_online():
    db = configure_db('ddl')
    with db.engine.connect() as connection:
        context.configure(connection=connection)
        with connection.begin() as trans:
            context.run_migrations()
            trans.commit()
Ejemplo n.º 4
0
def main(argv, _db_rw=None, _redis_client=None):  # pragma: no cover
    parser = argparse.ArgumentParser(
        prog=argv[0], description='Load/import cell data.')
    parser.add_argument('--datatype', default='ocid',
                        help='Type of the data file, cell or ocid')
    parser.add_argument('--filename',
                        help='Path to the gzipped csv file.')

    args = parser.parse_args(argv[1:])
    if not args.filename:
        parser.print_help()
        sys.exit(1)

    filename = os.path.abspath(args.filename)
    if not os.path.isfile(filename):
        print('File not found.')
        sys.exit(1)

    datatype = args.datatype
    if datatype not in ('cell', 'ocid'):
        print('Unknown data type.')
        sys.exit(1)

    configure_logging()
    app_config = read_config()
    db = configure_db(app_config.get('database', 'rw_url'), _db=_db_rw)
    redis_client = configure_redis(
        app_config.get('cache', 'cache_url'), _client=_redis_client)

    load_file(db, redis_client, datatype, filename)
Ejemplo n.º 5
0
def init_worker(celery_app,
                app_config,
                _db_rw=None,
                _db_ro=None,
                _geoip_db=None,
                _raven_client=None,
                _redis_client=None,
                _stats_client=None):
    """
    Configure the passed in celery app, usually stored in
    :data:`ichnaea.async.app.celery_app`.

    Does connection, settings and queue setup. Attaches some
    additional functionality to the :class:`celery.Celery` instance.

    This is executed inside each forked worker process.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param _db_ro: Ignored, read-only database connection isn't used.
    """

    # make config file settings available
    celery_app.settings = app_config.asdict()

    # configure outside connections
    celery_app.db_rw = configure_db(app_config.get('database', 'rw_url'),
                                    _db=_db_rw)

    celery_app.raven_client = raven_client = configure_raven(
        app_config.get('sentry', 'dsn'),
        transport='threaded',
        _client=_raven_client)

    celery_app.redis_client = redis_client = configure_redis(
        app_config.get('cache', 'cache_url'), _client=_redis_client)

    celery_app.stats_client = configure_stats(app_config,
                                              _client=_stats_client)

    celery_app.geoip_db = configure_geoip(app_config.get('geoip', 'db_path'),
                                          raven_client=raven_client,
                                          _client=_geoip_db)

    # configure data / export queues
    celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES])

    celery_app.data_queues = data_queues = configure_data(redis_client)
    for queue in data_queues.values():
        if queue.monitor_name:
            all_queues.add(queue.monitor_name)

    celery_app.export_queues = configure_export(redis_client, app_config)
    for queue in celery_app.export_queues.values():
        if queue.monitor_name:
            all_queues.add(queue.monitor_name)
Ejemplo n.º 6
0
def init_worker(
    celery_app,
    app_config,
    _db_rw=None,
    _db_ro=None,
    _geoip_db=None,
    _raven_client=None,
    _redis_client=None,
    _stats_client=None,
):
    """
    Configure the passed in celery app, usually stored in
    :data:`ichnaea.async.app.celery_app`.

    Does connection, settings and queue setup. Attaches some
    additional functionality to the :class:`celery.Celery` instance.

    This is executed inside each forked worker process.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param _db_ro: Ignored, read-only database connection isn't used.
    """

    # make config file settings available
    celery_app.settings = app_config.asdict()

    # configure outside connections
    celery_app.db_rw = configure_db(app_config.get("ichnaea", "db_master"), _db=_db_rw)

    celery_app.raven_client = raven_client = configure_raven(
        app_config.get("ichnaea", "sentry_dsn"), transport="threaded", _client=_raven_client
    )

    celery_app.redis_client = redis_client = configure_redis(
        app_config.get("ichnaea", "redis_url"), _client=_redis_client
    )

    celery_app.stats_client = configure_stats(app_config.get("ichnaea", "statsd_host"), _client=_stats_client)

    celery_app.geoip_db = configure_geoip(
        app_config.get("ichnaea", "geoip_db_path"), raven_client=raven_client, _client=_geoip_db
    )

    # configure data / export queues
    celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES])

    celery_app.data_queues = data_queues = configure_data(redis_client)
    for queue in data_queues.values():
        if queue.monitor_name:
            all_queues.add(queue.monitor_name)

    celery_app.export_queues = configure_export(redis_client, app_config)
    for queue in celery_app.export_queues.values():
        if queue.monitor_name:
            all_queues.add(queue.monitor_name)
Ejemplo n.º 7
0
def run_migrations_online():
    # Create a database connection using SQLALCHEMY_URL defined in the
    # environment.
    db_url = get_sqlalchemy_url()
    db = configure_db(uri=db_url, pool=False)

    with db.engine.connect() as connection:
        context.configure(connection=connection)
        with context.begin_transaction():
            context.run_migrations()
 def session2(self, database):
     # Create a second independent database session.
     db = configure_db('ro')
     session = db.session()
     try:
         yield session
         session.rollback()
     finally:
         session.close()
         db.close()
Ejemplo n.º 9
0
 def session2(self, database):
     # Create a second independent database session.
     db = configure_db('ro')
     session = db.session()
     try:
         yield session
         session.rollback()
     finally:
         session.close()
         db.close()
Ejemplo n.º 10
0
def init_worker(celery_app,
                app_config,
                _db_rw=None,
                _db_ro=None,
                _geoip_db=None,
                _raven_client=None,
                _redis_client=None,
                _stats_client=None):
    # currently neither a db_ro nor geoip_db are set up

    # make config file settings available
    celery_app.settings = app_config.asdict()

    # configure data / export queues
    celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES])

    celery_app.data_queues = data_queues = {
        'cell_area_update': 'update_cell_lac',
    }
    for value in data_queues.values():
        all_queues.add(value)

    celery_app.export_queues = export_queues = {}
    for section_name in app_config.sections():
        if section_name.startswith('export:'):
            section = app_config.get_map(section_name)
            name = section_name.split(':')[1]
            queue_name = EXPORT_QUEUE_PREFIX + name
            export_queues[name] = {
                'redis_key': queue_name,
            }
            all_queues.add(queue_name)
            for key, value in section.items():
                if key == 'batch':
                    export_queues[name][key] = int(value)
                else:
                    export_queues[name][key] = value

    # configure outside connections
    celery_app.db_rw = configure_db(app_config.get('ichnaea', 'db_master'),
                                    _db=_db_rw)

    celery_app.raven_client = configure_raven(app_config.get(
        'ichnaea', 'sentry_dsn'),
                                              transport='threaded',
                                              _client=_raven_client)

    celery_app.redis_client = configure_redis(app_config.get(
        'ichnaea', 'redis_url'),
                                              _client=_redis_client)

    celery_app.stats_client = configure_stats(app_config.get(
        'ichnaea', 'statsd_host'),
                                              _client=_stats_client)
Ejemplo n.º 11
0
def setup_database():
    # Create it if it doesn't exist
    try:
        create_db()
    except ProgrammingError:
        pass

    # Clean up the tables and set them up
    db = configure_db(uri=get_sqlalchemy_url())
    cleanup_tables(db.engine)
    setup_tables(db.engine)
    db.close()
Ejemplo n.º 12
0
def main(argv, _db=None, _dump_file=dump_file):
    parser = argparse.ArgumentParser(prog=argv[0],
                                     description='Dump/export data.')
    parser.add_argument('--datatype',
                        required=True,
                        help='Type of the data file, blue, cell or wifi')
    parser.add_argument('--filename',
                        required=True,
                        help='Path to the csv.gz export file.')
    parser.add_argument('--lat',
                        default=None,
                        help='The center latitude of the desired area.')
    parser.add_argument('--lon',
                        default=None,
                        help='The center longitude of the desired area.')
    parser.add_argument('--radius',
                        default=None,
                        help='The radius of the desired area.')

    args = parser.parse_args(argv[1:])
    if not args.filename:  # pragma: no cover
        parser.print_help()
        return 1

    filename = os.path.abspath(os.path.expanduser(args.filename))
    if os.path.isfile(filename):  # pragma: no cover
        print('File already exists.')
        return 1

    datatype = args.datatype
    if datatype not in ('blue', 'cell', 'wifi'):  # pragma: no cover
        print('Unknown data type.')
        return 1

    lat, lon, radius = (None, None, None)
    if (args.lat is not None and args.lon is not None
            and args.radius is not None):
        lat = float(args.lat)
        lon = float(args.lon)
        radius = int(args.radius)

    configure_logging()

    db = configure_db('ro', _db=_db)
    with db_worker_session(db, commit=False) as session:
        exit_code = _dump_file(datatype,
                               session,
                               filename,
                               lat=lat,
                               lon=lon,
                               radius=radius)
    return exit_code
Ejemplo n.º 13
0
def main(argv, _db=None, _dump_file=dump_file):
    parser = argparse.ArgumentParser(prog=argv[0],
                                     description="Dump/export data.")
    parser.add_argument("--datatype",
                        required=True,
                        help="Type of the data file, blue, cell or wifi")
    parser.add_argument("--filename",
                        required=True,
                        help="Path to the csv.gz export file.")
    parser.add_argument("--lat",
                        default=None,
                        help="The center latitude of the desired area.")
    parser.add_argument("--lon",
                        default=None,
                        help="The center longitude of the desired area.")
    parser.add_argument("--radius",
                        default=None,
                        help="The radius of the desired area.")

    args = parser.parse_args(argv[1:])
    if not args.filename:
        parser.print_help()
        return 1

    filename = os.path.abspath(os.path.expanduser(args.filename))
    if os.path.isfile(filename):
        print("File already exists.")
        return 1

    datatype = args.datatype
    if datatype not in ("blue", "cell", "wifi"):
        print("Unknown data type.")
        return 1

    lat, lon, radius = (None, None, None)
    if args.lat is not None and args.lon is not None and args.radius is not None:
        lat = float(args.lat)
        lon = float(args.lon)
        radius = int(args.radius)

    configure_logging()

    db = configure_db("ro", _db=_db, pool=False)
    with db_worker_session(db, commit=False) as session:
        exit_code = _dump_file(datatype,
                               session,
                               filename,
                               lat=lat,
                               lon=lon,
                               radius=radius)
    return exit_code
Ejemplo n.º 14
0
def show_api_key_details(ctx, key):
    """Print api key details to stdout."""
    db = configure_db("rw")
    with db_worker_session(db) as session:
        row = session.query(ApiKey).filter(
            ApiKey.valid_key == key).one_or_none()
        if row:
            api_key = Key.from_obj(row)
        else:
            api_key = None

    if api_key:
        table = [[name, value] for name, value in api_key.as_dict().items()]
        print_table(table, delimiter=" : ", stream_write=click_echo_no_nl)
    else:
        click.echo(f"API key '{key}' does not exist")
Ejemplo n.º 15
0
def show_api_key_details(key):
    """Print api key details to stdout."""
    db = configure_db("rw")
    with db_worker_session(db) as session:
        columns = ApiKey.__table__.columns
        fields = [getattr(columns, f) for f in API_KEY_COLUMN_NAMES]
        row = (session.execute(
            select(fields).where(columns.valid_key == key))).fetchone()
        if row is not None:
            key = Key(**dict(row.items()))
        else:
            key = None
    table = []
    for field in API_KEY_COLUMN_NAMES:
        table.append([field, getattr(key, field, "")])

    print_table(table, " : ")
Ejemplo n.º 16
0
def export_file(filename, tablename, _db=None, _session=None):
    today = util.utcnow().date()
    one_year_ago = today - timedelta(days=365)
    one_year_ago = one_year_ago.strftime('%Y-%m-%d')
    # this is executed in a worker process
    stmt = text('''\
SELECT
`grid`, CAST(ROUND(DATEDIFF(CURDATE(), `modified`) / 30) AS UNSIGNED) as `num`
FROM {tablename}
WHERE modified >= '{modified}'
LIMIT :limit OFFSET :offset
'''.format(tablename=tablename, modified=one_year_ago).replace('\n', ' '))
    db = configure_db('ro', _db=_db)

    offset = 0
    limit = 200000

    result_rows = 0
    with util.gzip_open(filename, 'w', compresslevel=2) as fd:
        with db_worker_session(db, commit=False) as session:
            if _session is not None:
                # testing hook
                session = _session
            while True:
                result = session.execute(
                    stmt.bindparams(limit=limit, offset=offset))
                rows = result.fetchall()
                result.close()
                if not rows:
                    break

                lines = []
                extend = lines.extend
                for row in rows:
                    lat, lon = decode_datamap_grid(row.grid)
                    extend(random_points(lat, lon, row.num))

                fd.writelines(lines)
                result_rows += len(lines)
                offset += limit

    if not result_rows:
        os.remove(filename)

    db.close()
    return result_rows
Ejemplo n.º 17
0
def init_worker(celery_app, app_config,
                _db_rw=None, _db_ro=None, _geoip_db=None,
                _raven_client=None, _redis_client=None, _stats_client=None):
    # currently neither a db_ro nor geoip_db are set up

    # make config file settings available
    celery_app.settings = app_config.asdict()

    # configure data / export queues
    celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES])

    celery_app.data_queues = data_queues = {
        'cell_area_update': 'update_cell_lac',
    }
    for value in data_queues.values():
        all_queues.add(value)

    celery_app.export_queues = export_queues = {}
    for section_name in app_config.sections():
        if section_name.startswith('export:'):
            section = app_config.get_map(section_name)
            name = section_name.split(':')[1]
            queue_name = EXPORT_QUEUE_PREFIX + name
            export_queues[name] = {
                'redis_key': queue_name,
            }
            all_queues.add(queue_name)
            for key, value in section.items():
                if key == 'batch':
                    export_queues[name][key] = int(value)
                else:
                    export_queues[name][key] = value

    # configure outside connections
    celery_app.db_rw = configure_db(
        app_config.get('ichnaea', 'db_master'), _db=_db_rw)

    celery_app.raven_client = configure_raven(
        app_config.get('ichnaea', 'sentry_dsn'), _client=_raven_client)

    celery_app.redis_client = configure_redis(
        app_config.get('ichnaea', 'redis_url'), _client=_redis_client)

    celery_app.stats_client = configure_stats(
        app_config.get('ichnaea', 'statsd_host'), _client=_stats_client)
Ejemplo n.º 18
0
def export_file(filename, tablename, _db=None, _session=None):
    # this is executed in a worker process
    stmt = text('''\
SELECT
`grid`, CAST(ROUND(DATEDIFF(CURDATE(), `modified`) / 30) AS UNSIGNED) as `num`
FROM {tablename}
WHERE `grid` > :grid
ORDER BY `grid`
LIMIT :limit
'''.format(tablename=tablename).replace('\n', ' '))

    db = configure_db('ro', transport='sync', _db=_db)
    min_grid = b''
    limit = 200000

    result_rows = 0
    with util.gzip_open(filename, 'w', compresslevel=2) as fd:
        with db_worker_session(db, commit=False) as session:
            if _session is not None:
                # testing hook
                session = _session
            while True:
                result = session.execute(
                    stmt.bindparams(limit=limit, grid=min_grid))
                rows = result.fetchall()
                result.close()
                if not rows:
                    break

                lines = []
                extend = lines.extend
                for row in rows:
                    lat, lon = decode_datamap_grid(row.grid)
                    extend(random_points(lat, lon, row.num))

                fd.writelines(lines)
                result_rows += len(lines)
                min_grid = rows[-1].grid

    if not result_rows:
        os.remove(filename)

    db.close()
    return result_rows
Ejemplo n.º 19
0
def main(argv, _db_rw=None, _raven_client=None):
    parser = argparse.ArgumentParser(prog=argv[0],
                                     description='Initialize Ichnaea database')

    parser.add_argument('--alembic_ini',
                        help='Path to the alembic migration config.')
    parser.add_argument('--location_ini',
                        help='Path to the ichnaea app config.')
    parser.add_argument('--initdb',
                        action='store_true',
                        help='Initialize database')

    args = parser.parse_args(argv[1:])

    if args.initdb:
        # Either use explicit config file location or fallback
        # on environment variable or finally file in current directory
        if not args.location_ini:
            location_ini = os.environ.get('ICHNAEA_CFG', 'location.ini')
        else:
            location_ini = args.location_ini
        location_ini = os.path.abspath(location_ini)
        location_cfg = read_config(filename=location_ini)

        # Either use explicit config file location or fallback
        # to a file in the same directory as the location.ini
        if not args.alembic_ini:
            alembic_ini = os.path.join(os.path.dirname(location_ini),
                                       'alembic.ini')
        else:
            alembic_ini = args.alembic_ini
        alembic_ini = os.path.abspath(alembic_ini)
        alembic_cfg = Config(alembic_ini)
        alembic_section = alembic_cfg.get_section('alembic')

        db_rw = configure_db(alembic_section['sqlalchemy.url'], _db=_db_rw)
        configure_raven(location_cfg.get('sentry', 'dsn'),
                        transport='sync',
                        _client=_raven_client)

        engine = db_rw.engine
        create_schema(engine, alembic_cfg, location_cfg)
    else:
        parser.print_help()
Ejemplo n.º 20
0
def main(argv, _db_rw=None, _raven_client=None):
    parser = argparse.ArgumentParser(
        prog=argv[0], description='Initialize Ichnaea database')

    parser.add_argument('--alembic_ini',
                        help='Path to the alembic migration config.')
    parser.add_argument('--location_ini',
                        help='Path to the ichnaea app config.')
    parser.add_argument('--initdb', action='store_true',
                        help='Initialize database')

    args = parser.parse_args(argv[1:])

    if args.initdb:
        # Either use explicit config file location or fallback
        # on environment variable or finally file in current directory
        if not args.location_ini:
            location_ini = os.environ.get('ICHNAEA_CFG', 'location.ini')
        else:
            location_ini = args.location_ini
        location_ini = os.path.abspath(location_ini)
        location_cfg = read_config(filename=location_ini)

        # Either use explicit config file location or fallback
        # to a file in the same directory as the location.ini
        if not args.alembic_ini:
            alembic_ini = os.path.join(
                os.path.dirname(location_ini), 'alembic.ini')
        else:
            alembic_ini = args.alembic_ini
        alembic_ini = os.path.abspath(alembic_ini)
        alembic_cfg = Config(alembic_ini)
        alembic_section = alembic_cfg.get_section('alembic')

        db_rw = configure_db(
            alembic_section['sqlalchemy.url'], _db=_db_rw)
        configure_raven(
            location_cfg.get('sentry', 'dsn'),
            transport='sync', _client=_raven_client)

        engine = db_rw.engine
        create_schema(engine, alembic_cfg, location_cfg)
    else:
        parser.print_help()
Ejemplo n.º 21
0
def export_file(filename, tablename, _db=None, _session=None):
    # this is executed in a worker process
    stmt = text("""\
SELECT
`grid`, CAST(ROUND(DATEDIFF(CURDATE(), `modified`) / 30) AS UNSIGNED) as `num`
FROM {tablename}
WHERE `grid` > :grid
ORDER BY `grid`
LIMIT :limit
""".format(tablename=tablename).replace("\n", " "))

    db = configure_db("ro", _db=_db, pool=False)
    min_grid = b""
    limit = 200000

    result_rows = 0
    with util.gzip_open(filename, "w", compresslevel=2) as fd:
        with db_worker_session(db, commit=False) as session:
            if _session is not None:
                # testing hook
                session = _session
            while True:
                result = session.execute(
                    stmt.bindparams(limit=limit, grid=min_grid))
                rows = result.fetchall()
                result.close()
                if not rows:
                    break

                lines = []
                extend = lines.extend
                for row in rows:
                    lat, lon = decode_datamap_grid(row.grid)
                    extend(random_points(lat, lon, row.num))

                fd.writelines(lines)
                result_rows += len(lines)
                min_grid = rows[-1].grid

    if not result_rows:
        os.remove(filename)

    db.close()
    return result_rows
Ejemplo n.º 22
0
def main(argv, _db=None, _dump_file=dump_file):
    parser = argparse.ArgumentParser(
        prog=argv[0], description='Dump/export data.')
    parser.add_argument('--datatype', required=True,
                        help='Type of the data file, blue, cell or wifi')
    parser.add_argument('--filename', required=True,
                        help='Path to the csv.gz export file.')
    parser.add_argument('--lat', default=None,
                        help='The center latitude of the desired area.')
    parser.add_argument('--lon', default=None,
                        help='The center longitude of the desired area.')
    parser.add_argument('--radius', default=None,
                        help='The radius of the desired area.')

    args = parser.parse_args(argv[1:])
    if not args.filename:  # pragma: no cover
        parser.print_help()
        return 1

    filename = os.path.abspath(os.path.expanduser(args.filename))
    if os.path.isfile(filename):  # pragma: no cover
        print('File already exists.')
        return 1

    datatype = args.datatype
    if datatype not in ('blue', 'cell', 'wifi'):  # pragma: no cover
        print('Unknown data type.')
        return 1

    lat, lon, radius = (None, None, None)
    if (args.lat is not None and
            args.lon is not None and args.radius is not None):
        lat = float(args.lat)
        lon = float(args.lon)
        radius = int(args.radius)

    configure_logging()

    db = configure_db('ro', transport='sync', _db=_db)
    with db_worker_session(db, commit=False) as session:
        exit_code = _dump_file(
            datatype, session, filename, lat=lat, lon=lon, radius=radius)
    return exit_code
Ejemplo n.º 23
0
def create_api_key(key):
    """Create a new api key."""
    key = key or str(uuid.uuid4())

    db = configure_db("rw")
    with db_worker_session(db) as session:
        try:
            session.execute(
                insert(ApiKey.__table__).values(
                    valid_key=key,
                    allow_fallback=False,
                    allow_locate=True,
                    allow_region=True,
                    store_sample_locate=100,
                    store_sample_submit=100,
                ))
            print("Created API key: %r" % key)
        except IntegrityError:
            print("API key %r exists" % key)
Ejemplo n.º 24
0
def export_file(db_url, filename, tablename, _db_rw=None, _session=None):
    # this is executed in a worker process
    stmt = text('''\
SELECT
`grid`, CAST(ROUND(DATEDIFF(CURDATE(), `modified`) / 30) AS UNSIGNED) as `num`
FROM {tablename}
LIMIT :limit OFFSET :offset
'''.format(tablename=tablename).replace('\n', ' '))
    db = configure_db(db_url, _db=_db_rw)

    offset = 0
    limit = 200000

    result_rows = 0
    with util.gzip_open(filename, 'w', compresslevel=2) as fd:
        with db_worker_session(db, commit=False) as session:
            if _session is not None:
                # testing hook
                session = _session
            while True:
                result = session.execute(
                    stmt.bindparams(limit=limit, offset=offset))
                rows = result.fetchall()
                result.close()
                if not rows:
                    break

                lines = []
                extend = lines.extend
                for row in rows:
                    lat, lon = decode_datamap_grid(row.grid)
                    extend(random_points(lat, lon, row.num))

                fd.writelines(lines)
                result_rows += len(lines)
                offset += limit

    if not result_rows:
        os.remove(filename)

    db.engine.pool.dispose()
    return result_rows
Ejemplo n.º 25
0
def init_worker(celery_app,
                _db_rw=None, _db_ro=None, _geoip_db=None,
                _raven_client=None, _redis_client=None, _stats_client=None):
    """
    Configure the passed in celery app, usually stored in
    :data:`ichnaea.async.app.celery_app`.

    Does connection, settings and queue setup. Attaches some
    additional functionality to the :class:`celery.Celery` instance.

    This is executed inside each forked worker process.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param _db_ro: Ignored, read-only database connection isn't used.
    """

    # configure outside connections
    celery_app.db_rw = configure_db(
        celery_app.app_config.get('database', 'rw_url'), _db=_db_rw)

    celery_app.raven_client = raven_client = configure_raven(
        celery_app.app_config.get('sentry', 'dsn'),
        transport='threaded', _client=_raven_client)

    celery_app.redis_client = redis_client = configure_redis(
        celery_app.app_config.get('cache', 'cache_url'), _client=_redis_client)

    celery_app.stats_client = configure_stats(
        celery_app.app_config, _client=_stats_client)

    celery_app.geoip_db = configure_geoip(
        celery_app.app_config.get('geoip', 'db_path'),
        raven_client=raven_client, _client=_geoip_db)

    # configure data queues
    celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES])

    celery_app.data_queues = data_queues = configure_data(redis_client)
    all_queues = all_queues.union(
        set([queue.key for queue in data_queues.values() if queue.key]))
Ejemplo n.º 26
0
def list_api_keys(ctx):
    """List all api keys in db."""
    show_fields = [
        "valid_key", "allow_fallback", "allow_locate", "allow_region"
    ]

    db = configure_db("rw")
    with db_worker_session(db) as session:
        columns = ApiKey.__table__.columns
        fields = [getattr(columns, f) for f in show_fields]
        rows = session.execute(select(fields)).fetchall()

    click.echo("%d api keys." % len(rows))
    if rows:
        # Add header row
        table = [show_fields]
        # Add rest of the rows; the columns are in the order of show_fields so we
        # don't have to do any re-ordering
        table.extend(rows)
        print_table(table, stream_write=click_echo_no_nl)
Ejemplo n.º 27
0
    def broken_app(self, http_session, raven):
        # Create database connections to the discard port.
        db = configure_db("rw", uri="mysql+pymysql://none:[email protected]:9/none")

        # Create a broken GeoIP database.
        geoip_db = GeoIPNull()

        # Create a broken Redis client.
        redis_client = configure_redis("redis://127.0.0.1:9/15")

        app = _make_app(
            _db=db,
            _geoip_db=geoip_db,
            _http_session=http_session,
            _raven_client=raven,
            _redis_client=redis_client,
        )
        yield app

        db.close()
        geoip_db.close()
        redis_client.close()
def init_worker(celery_app,
                _db=None,
                _geoip_db=None,
                _raven_client=None,
                _redis_client=None,
                _stats_client=None):
    """
    Configure the passed in celery app, usually stored in
    :data:`ichnaea.async.app.celery_app`.

    Does connection, settings and queue setup. Attaches some
    additional functionality to the :class:`celery.Celery` instance.

    This is executed inside each forked worker process.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.
    """

    # configure outside connections
    celery_app.db = configure_db('rw', _db=_db)

    celery_app.raven_client = raven_client = configure_raven(
        transport='threaded', _client=_raven_client)

    celery_app.redis_client = redis_client = configure_redis(
        _client=_redis_client)

    celery_app.stats_client = configure_stats(_client=_stats_client)

    celery_app.geoip_db = configure_geoip(raven_client=raven_client,
                                          _client=_geoip_db)

    # configure data queues
    celery_app.all_queues = all_queues = set([q.name for q in TASK_QUEUES])

    celery_app.data_queues = data_queues = configure_data(redis_client)
    all_queues = all_queues.union(
        set([queue.key for queue in data_queues.values() if queue.key]))
Ejemplo n.º 29
0
def init_worker(celery_app, app_config,
                _db_rw=None, _db_ro=None, _geoip_db=None,
                _raven_client=None, _redis_client=None, _stats_client=None):
    # currently db_ro is not set up

    # make config file settings available
    celery_app.settings = app_config.asdict()

    # configure outside connections
    celery_app.db_rw = configure_db(
        app_config.get('ichnaea', 'db_master'), _db=_db_rw)

    celery_app.raven_client = raven_client = configure_raven(
        app_config.get('ichnaea', 'sentry_dsn'),
        transport='threaded', _client=_raven_client)

    celery_app.redis_client = redis_client = configure_redis(
        app_config.get('ichnaea', 'redis_url'), _client=_redis_client)

    celery_app.stats_client = configure_stats(
        app_config.get('ichnaea', 'statsd_host'), _client=_stats_client)

    celery_app.geoip_db = configure_geoip(
        app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client,
        _client=_geoip_db)

    # configure data / export queues
    celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES])

    celery_app.data_queues = data_queues = configure_data(redis_client)
    for queue in data_queues.values():
        if queue.monitor_name:
            all_queues.add(queue.monitor_name)

    celery_app.export_queues = configure_export(redis_client, app_config)
    for queue in celery_app.export_queues.values():
        if queue.monitor_name:
            all_queues.add(queue.monitor_name)
Ejemplo n.º 30
0
    def broken_app(self, http_session, raven, stats):
        # Create database connections to the discard port.
        db = configure_db(uri='mysql+pymysql://none:[email protected]:9/none')

        # Create a broken GeoIP database.
        geoip_db = GeoIPNull()

        # Create a broken Redis client.
        redis_client = configure_redis('redis://127.0.0.1:9/15')

        app = _make_app(
            _db=db,
            _geoip_db=geoip_db,
            _http_session=http_session,
            _raven_client=raven,
            _redis_client=redis_client,
            _stats_client=stats,
        )
        yield app

        db.close()
        geoip_db.close()
        redis_client.close()
Ejemplo n.º 31
0
def create_api_key(ctx, maxreq, key):
    """Create a new api key.

    If KEY is not specified, it uses a uuid4.

    """
    key = key or str(uuid.uuid4())

    db = configure_db("rw")
    with db_worker_session(db) as session:
        try:
            session.execute(
                insert(ApiKey.__table__).values(
                    valid_key=key,
                    maxreq=maxreq,
                    allow_fallback=False,
                    allow_locate=True,
                    allow_region=True,
                    store_sample_locate=100,
                    store_sample_submit=100,
                ))
            click.echo("Created API key: %r" % key)
        except IntegrityError:
            click.echo("API key %r exists" % key)
Ejemplo n.º 32
0
def main(app_config, ping_connections=False,
         _db_rw=None, _db_ro=None, _geoip_db=None, _http_session=None,
         _raven_client=None, _redis_client=None, _stats_client=None,
         _country_searcher=None, _position_searcher=None):
    """
    Configure the web app stored in :data:`ichnaea.webapp.app._APP`.

    Does connection, logging and view config setup. Attaches some
    additional functionality to the :class:`pyramid.registry.Registry`
    instance.

    At startup ping all outbound connections like the database
    once, to ensure they are actually up and responding.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param app_config: The parsed application ini.
    :type app_config: :class:`ichnaea.config.Config`

    :param ping_connections: If True, ping and test outside connections.
    :type ping_connections: bool

    :returns: A configured WSGI app, the result of calling
              :meth:`pyramid.config.Configurator.make_wsgi_app`.
    """

    configure_logging()

    # make config file settings available
    config = Configurator(settings=app_config.asdict())

    # add support for pt templates
    config.include('pyramid_chameleon')

    # add a config setting to skip logging for some views
    config.registry.skip_logging = set()

    configure_api(config)
    configure_content(config)
    configure_monitor(config)

    # configure outside connections
    registry = config.registry

    registry.db_rw = configure_db(
        app_config.get('database', 'rw_url'), _db=_db_rw)
    registry.db_ro = configure_db(
        app_config.get('database', 'ro_url'), _db=_db_ro)

    registry.raven_client = raven_client = configure_raven(
        app_config.get('sentry', 'dsn'),
        transport='gevent', _client=_raven_client)

    registry.redis_client = redis_client = configure_redis(
        app_config.get('cache', 'cache_url'), _client=_redis_client)

    registry.stats_client = stats_client = configure_stats(
        app_config, _client=_stats_client)

    registry.http_session = configure_http_session(_session=_http_session)

    registry.geoip_db = geoip_db = configure_geoip(
        app_config.get('geoip', 'db_path'), raven_client=raven_client,
        _client=_geoip_db)

    for name, func, default in (('country_searcher',
                                 configure_country_searcher,
                                 _country_searcher),
                                ('position_searcher',
                                 configure_position_searcher,
                                 _position_searcher)):
        searcher = func(app_config,
                        geoip_db=geoip_db, raven_client=raven_client,
                        redis_client=redis_client, stats_client=stats_client,
                        _searcher=default)
        setattr(registry, name, searcher)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_rw_session, property=True)
    config.add_request_method(db_ro_session, property=True)

    # Add special JSON renderer with nicer float representation
    config.add_renderer('floatjson', floatjson.FloatJSONRenderer())

    # freeze skip logging set
    config.registry.skip_logging = frozenset(config.registry.skip_logging)

    # Should we try to initialize and establish the outbound connections?
    if ping_connections:  # pragma: no cover
        registry.db_ro.ping()
        registry.redis_client.ping()

    return config.make_wsgi_app()
Ejemplo n.º 33
0
def _make_db(uri=SQLURI):
    return configure_db(uri)
Ejemplo n.º 34
0
def main(global_config,
         app_config,
         init=False,
         _db_rw=None,
         _db_ro=None,
         _geoip_db=None,
         _raven_client=None,
         _redis_client=None,
         _stats_client=None):

    configure_logging()

    # make config file settings available
    config = Configurator(settings=app_config.asdict())

    # add support for pt templates
    config.include('pyramid_chameleon')

    configure_content(config)
    configure_service(config)

    # configure outside connections
    registry = config.registry

    registry.db_rw = configure_db(app_config.get('ichnaea', 'db_master'),
                                  _db=_db_rw)
    registry.db_ro = configure_db(app_config.get('ichnaea', 'db_slave'),
                                  _db=_db_ro)

    registry.raven_client = raven_client = configure_raven(
        app_config.get('ichnaea', 'sentry_dsn'),
        transport='gevent',
        _client=_raven_client)

    registry.redis_client = configure_redis(app_config.get(
        'ichnaea', 'redis_url'),
                                            _client=_redis_client)

    registry.stats_client = configure_stats(app_config.get(
        'ichnaea', 'statsd_host'),
                                            _client=_stats_client)

    registry.geoip_db = configure_geoip(app_config.get('ichnaea',
                                                       'geoip_db_path'),
                                        raven_client=raven_client,
                                        _client=_geoip_db)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_rw_session, property=True)
    config.add_request_method(db_ro_session, property=True)

    # replace json renderer with custom json variant
    config.add_renderer('json', customjson.Renderer())

    # Should we try to initialize and establish the outbound connections?
    if init:  # pragma: no cover
        registry.db_ro.ping()
        registry.redis_client.ping()
        registry.stats_client.ping()

    return config.make_wsgi_app()
Ejemplo n.º 35
0
def main(ping_connections=False,
         _db=None, _geoip_db=None, _http_session=None,
         _raven_client=None, _redis_client=None, _stats_client=None,
         _position_searcher=None, _region_searcher=None):
    """
    Configure the web app stored in :data:`ichnaea.webapp.app._APP`.

    Does connection, logging and view config setup. Attaches some
    additional functionality to the :class:`pyramid.registry.Registry`
    instance.

    At startup ping all outbound connections like the database
    once, to ensure they are actually up and responding.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param ping_connections: If True, ping and test outside connections.
    :type ping_connections: bool

    :returns: A configured WSGI app, the result of calling
              :meth:`pyramid.config.Configurator.make_wsgi_app`.
    """

    configure_logging()

    config = Configurator()

    # add support for pt templates
    config.include('pyramid_chameleon')

    # add a config setting to skip logging for some views
    config.registry.skip_logging = set()

    configure_api(config)
    configure_content(config)
    configure_monitor(config)

    # configure outside connections
    registry = config.registry

    registry.db = configure_db('ro', _db=_db)

    registry.raven_client = raven_client = configure_raven(
        transport='gevent', _client=_raven_client)

    registry.redis_client = redis_client = configure_redis(
        _client=_redis_client)

    registry.stats_client = stats_client = configure_stats(
        _client=_stats_client)

    registry.http_session = configure_http_session(_session=_http_session)

    registry.geoip_db = geoip_db = configure_geoip(
        raven_client=raven_client, _client=_geoip_db)

    # Needs to be the exact same as the *_incoming entries in async.config.
    registry.data_queues = data_queues = {
        'update_incoming': DataQueue('update_incoming', redis_client,
                                     batch=100, compress=True),
    }

    for name, func, default in (('position_searcher',
                                 configure_position_searcher,
                                 _position_searcher),
                                ('region_searcher',
                                 configure_region_searcher,
                                 _region_searcher)):
        searcher = func(geoip_db=geoip_db, raven_client=raven_client,
                        redis_client=redis_client, stats_client=stats_client,
                        data_queues=data_queues, _searcher=default)
        setattr(registry, name, searcher)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_session, property=True)

    # freeze skip logging set
    config.registry.skip_logging = frozenset(config.registry.skip_logging)

    # Should we try to initialize and establish the outbound connections?
    if ping_connections:  # pragma: no cover
        registry.db.ping()
        registry.redis_client.ping()

    return config.make_wsgi_app()
Ejemplo n.º 36
0
def export_to_csv(filename, csv_dir, tablename, row_limit=None, file_limit=None):
    """
    Export a datamap table to a CSV file.

    :param filename: An output file ending in .csv
    :param csv_dir: The output directory
    :param tablename: The name of the datamap table to export
    :param row_limit: The number of rows to fetch at a time
    :param file_limit: The number of output rows before rotating files
    :return: A tuple (rows exported, files created)

    Each database row is turned into 0 to 6 similar CSV rows by
    random_points(), based on how recently they were recorded.

    If file_limit is not reached, the output file will the filename.
    If file_limit is reached, the output files will have a serial number and
    be based on the filename. For example, "map.csv" will become "map_0001.csv",
    "map_0002.csv", etc.
    """
    stmt = text(
        """\
SELECT
`grid`, CAST(ROUND(DATEDIFF(CURDATE(), `modified`) / 30) AS UNSIGNED) as `num`
FROM {tablename}
WHERE `grid` > :grid
ORDER BY `grid`
LIMIT :limit
""".format(
            tablename=tablename
        ).replace(
            "\n", " "
        )
    )

    db = configure_db("ro", pool=False)
    min_grid = b""
    row_limit = row_limit or 200_000
    file_limit = file_limit or 10_000_000

    result_rows = 0
    file_path = os.path.join(csv_dir, filename)
    fd = open(file_path, "w")
    file_count = 1
    file_rows = 0
    orig_filename = filename
    orig_file_path = file_path
    assert filename.endswith(".csv")
    try:
        with db_worker_session(db, commit=False) as session:
            while True:
                result = session.execute(
                    stmt.bindparams(limit=row_limit, grid=min_grid)
                )
                rows = result.fetchall()
                result.close()
                if not rows:
                    break

                lines = []
                extend = lines.extend
                for row in rows:
                    lat, lon = decode_datamap_grid(row.grid)
                    extend(random_points(lat, lon, row.num))

                fd.writelines(lines)
                result_rows += len(lines)

                # Rotate the file when needed
                file_rows += len(lines)
                if result_rows >= file_limit:
                    fd.close()
                    file_count += 1
                    file_rows = 0
                    filename = "sub" + orig_filename.replace(
                        ".csv", f"_{file_count:04}.csv"
                    )
                    file_path = os.path.join(csv_dir, filename)
                    fd = open(file_path, "w")

                min_grid = rows[-1].grid
    finally:
        fd.close()

    if not file_rows:
        os.remove(file_path)
        file_count -= 1

    if file_count > 1:
        # Rename first file to serial CSV format
        filename = "sub" + orig_filename.replace(".csv", "_0001.csv")
        file_path = os.path.join(csv_dir, filename)
        os.rename(orig_file_path, file_path)

    db.close()
    return result_rows, file_count
Ejemplo n.º 37
0
def db(database):
    db = configure_db(uri=get_sqlalchemy_url())
    yield db
    db.close()
def setup_database():
    db = configure_db('ddl')
    cleanup_tables(db.engine)
    setup_tables(db.engine)
    db.close()
def db(database):
    db = configure_db('rw')
    yield db
    db.close()
Ejemplo n.º 40
0
def main(argv, _db_rw=None,
         _raven_client=None, _stats_client=None):
    # run for example via:
    # bin/location_map --create --upload --datamaps=/path/to/datamaps/ \
    #   --output=ichnaea/content/static/tiles/

    parser = argparse.ArgumentParser(
        prog=argv[0], description='Generate and upload datamap tiles.')

    parser.add_argument('--create', action='store_true',
                        help='Create tiles.')
    parser.add_argument('--upload', action='store_true',
                        help='Upload tiles to S3.')
    parser.add_argument('--concurrency', default=2,
                        help='How many concurrent render processes to use?')
    parser.add_argument('--datamaps',
                        help='Directory of the datamaps tools.')
    parser.add_argument('--output',
                        help='Optional directory for local tile output.')

    args = parser.parse_args(argv[1:])

    if args.create:
        conf = read_config()
        db = configure_db(
            conf.get('database', 'rw_url'), _db=_db_rw)
        raven_client = configure_raven(
            conf.get('sentry', 'dsn'),
            transport='sync', _client=_raven_client)
        stats_client = configure_stats(conf, _client=_stats_client)

        bucketname = conf.get('assets', 'bucket').strip('/')

        upload = False
        if args.upload:  # pragma: no cover
            upload = bool(args.upload)

        concurrency = 2
        if args.concurrency:
            concurrency = int(args.concurrency)

        datamaps = ''
        if args.datamaps:
            datamaps = os.path.abspath(args.datamaps)

        output = None
        if args.output:
            output = os.path.abspath(args.output)

        try:
            with stats_client.timed('datamaps', tags=['func:main']):
                generate(db, bucketname, raven_client, stats_client,
                         upload=upload,
                         concurrency=concurrency,
                         datamaps=datamaps,
                         output=output)
        except Exception:  # pragma: no cover
            raven_client.captureException()
            raise
    else:  # pragma: no cover
        parser.print_help()
Ejemplo n.º 41
0
def main(app_config, ping_connections=False,
         _db_rw=None, _db_ro=None, _geoip_db=None,
         _raven_client=None, _redis_client=None, _stats_client=None):
    """
    Configure the web app stored in :data:`ichnaea.webapp.app._APP`.

    Does connection, logging and view config setup. Attaches some
    additional functionality to the :class:`pyramid.registry.Registry`
    instance.

    At startup ping all outbound connections like the database
    once, to ensure they are actually up and responding.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param app_config: The parsed application ini.
    :type app_config: :class:`ichnaea.config.Config`

    :param ping_connections: If True, ping and test outside connections.
    :type ping_connections: bool

    :returns: A configured WSGI app, the result of calling
              :meth:`pyramid.config.Configurator.make_wsgi_app`.
    """

    configure_logging()

    # make config file settings available
    config = Configurator(settings=app_config.asdict())

    # add support for pt templates
    config.include('pyramid_chameleon')

    configure_api(config)
    configure_content(config)
    configure_monitor(config)

    # configure outside connections
    registry = config.registry

    registry.db_rw = configure_db(
        app_config.get('ichnaea', 'db_master'), _db=_db_rw)
    registry.db_ro = configure_db(
        app_config.get('ichnaea', 'db_slave'), _db=_db_ro)

    registry.raven_client = raven_client = configure_raven(
        app_config.get('ichnaea', 'sentry_dsn'),
        transport='gevent', _client=_raven_client)

    registry.redis_client = configure_redis(
        app_config.get('ichnaea', 'redis_url'), _client=_redis_client)

    registry.stats_client = configure_stats(
        app_config.get('ichnaea', 'statsd_host'), _client=_stats_client)

    registry.geoip_db = configure_geoip(
        app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client,
        _client=_geoip_db)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_rw_session, property=True)
    config.add_request_method(db_ro_session, property=True)

    # replace json renderer with custom json variant
    config.add_renderer('json', customjson.Renderer())

    # Should we try to initialize and establish the outbound connections?
    if ping_connections:  # pragma: no cover
        registry.db_ro.ping()
        registry.redis_client.ping()
        registry.stats_client.ping()

    return config.make_wsgi_app()
Ejemplo n.º 42
0
def db(database):
    db = configure_db('rw')
    yield db
    db.close()
Ejemplo n.º 43
0
def sync_db(database):
    db = configure_db('rw', transport='sync')
    yield db
    db.close()
def sync_db(database):
    db = configure_db('rw', transport='sync')
    yield db
    db.close()
Ejemplo n.º 45
0
def setup_database():
    db = configure_db('ddl')
    cleanup_tables(db.engine)
    setup_tables(db.engine)
    db.close()
Ejemplo n.º 46
0
def main(
    ping_connections=False,
    _db=None,
    _geoip_db=None,
    _http_session=None,
    _raven_client=None,
    _redis_client=None,
    _position_searcher=None,
    _region_searcher=None,
):
    """
    Configure the web app stored in :data:`ichnaea.webapp.app._APP`.

    Does connection, logging and view config setup. Attaches some
    additional functionality to the :class:`pyramid.registry.Registry`
    instance.

    At startup ping all outbound connections like the database
    once, to ensure they are actually up and responding.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param ping_connections: If True, ping and test outside connections.
    :type ping_connections: bool

    :returns: A configured WSGI app, the result of calling
              :meth:`pyramid.config.Configurator.make_wsgi_app`.
    """

    configure_logging()

    config = Configurator()
    check_config()

    # add support for pt templates
    config.include("pyramid_chameleon")

    # add a config setting to skip logging for some views
    config.registry.skip_logging = set()

    configure_api(config)
    configure_content(config)
    configure_monitor(config)

    # configure outside connections
    registry = config.registry

    registry.db = configure_db("ro", _db=_db)

    registry.raven_client = raven_client = configure_raven(
        transport="gevent", tags={"app": "webapp"}, _client=_raven_client
    )

    registry.redis_client = redis_client = configure_redis(_client=_redis_client)

    configure_stats()

    registry.http_session = configure_http_session(_session=_http_session)

    registry.geoip_db = geoip_db = configure_geoip(
        raven_client=raven_client, _client=_geoip_db
    )

    # Needs to be the exact same as the *_incoming entries in taskapp.config.
    registry.data_queues = data_queues = {
        "update_incoming": DataQueue(
            "update_incoming", redis_client, "report", batch=100, compress=True
        )
    }

    for name, func, default in (
        ("position_searcher", configure_position_searcher, _position_searcher),
        ("region_searcher", configure_region_searcher, _region_searcher),
    ):
        searcher = func(
            geoip_db=geoip_db,
            raven_client=raven_client,
            redis_client=redis_client,
            data_queues=data_queues,
            _searcher=default,
        )
        setattr(registry, name, searcher)

    config.add_tween("ichnaea.db.db_tween_factory", under=EXCVIEW)
    config.add_tween("ichnaea.log.log_tween_factory", under=EXCVIEW)
    config.add_request_method(db_session, property=True)

    # freeze skip logging set
    config.registry.skip_logging = frozenset(config.registry.skip_logging)

    # Should we try to initialize and establish the outbound connections?
    if ping_connections:
        with db_worker_session(registry.db, commit=False) as session:
            ping_session(session)
        registry.redis_client.ping()

    return config.make_wsgi_app()
Ejemplo n.º 47
0
def _make_db(uri=SQLURI):
    return configure_db(uri)
Ejemplo n.º 48
0
def main(app_config, ping_connections=False,
         _db_rw=None, _db_ro=None, _geoip_db=None, _http_session=None,
         _raven_client=None, _redis_client=None, _stats_client=None,
         _position_searcher=None, _region_searcher=None):
    """
    Configure the web app stored in :data:`ichnaea.webapp.app._APP`.

    Does connection, logging and view config setup. Attaches some
    additional functionality to the :class:`pyramid.registry.Registry`
    instance.

    At startup ping all outbound connections like the database
    once, to ensure they are actually up and responding.

    The parameters starting with an underscore are test-only hooks
    to provide pre-configured connection objects.

    :param app_config: The parsed application ini.
    :type app_config: :class:`ichnaea.config.Config`

    :param ping_connections: If True, ping and test outside connections.
    :type ping_connections: bool

    :returns: A configured WSGI app, the result of calling
              :meth:`pyramid.config.Configurator.make_wsgi_app`.
    """

    configure_logging()

    # make config file settings available
    config = Configurator(settings=app_config.asdict())

    # add support for pt templates
    config.include('pyramid_chameleon')

    # add a config setting to skip logging for some views
    config.registry.skip_logging = set()

    configure_api(config)
    configure_content(config)
    configure_monitor(config)

    # configure outside connections
    registry = config.registry

    registry.db_rw = configure_db(
        app_config.get('database', 'rw_url'), _db=_db_rw)
    registry.db_ro = configure_db(
        app_config.get('database', 'ro_url'), _db=_db_ro)

    registry.raven_client = raven_client = configure_raven(
        app_config.get('sentry', 'dsn'),
        transport='gevent', _client=_raven_client)

    registry.redis_client = redis_client = configure_redis(
        app_config.get('cache', 'cache_url'), _client=_redis_client)

    registry.stats_client = stats_client = configure_stats(
        app_config, _client=_stats_client)

    registry.http_session = configure_http_session(_session=_http_session)

    registry.geoip_db = geoip_db = configure_geoip(
        app_config.get('geoip', 'db_path'), raven_client=raven_client,
        _client=_geoip_db)

    for name, func, default in (('position_searcher',
                                 configure_position_searcher,
                                 _position_searcher),
                                ('region_searcher',
                                 configure_region_searcher,
                                 _region_searcher)):
        searcher = func(app_config,
                        geoip_db=geoip_db, raven_client=raven_client,
                        redis_client=redis_client, stats_client=stats_client,
                        _searcher=default)
        setattr(registry, name, searcher)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_ro_session, property=True)

    # Add special JSON renderer with nicer float representation
    config.add_renderer('floatjson', floatjson.FloatJSONRenderer())

    # Add text-as-JS renderer.
    config.add_renderer('js', renderers.JSRenderer())

    # freeze skip logging set
    config.registry.skip_logging = frozenset(config.registry.skip_logging)

    # Should we try to initialize and establish the outbound connections?
    if ping_connections:  # pragma: no cover
        registry.db_ro.ping()
        registry.redis_client.ping()

    return config.make_wsgi_app()
Ejemplo n.º 49
0
def db_ro(database):
    db = configure_db(SQLURI)
    yield db
    db.close()
Ejemplo n.º 50
0
def setup_database():
    db = configure_db(SQLURI)
    engine = db.engine
    cleanup_tables(engine)
    setup_tables(engine)
    db.close()