def main(global_config, app_config, init=False, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): configure_logging() # make config file settings available config = Configurator(settings=app_config.asdict()) # add support for pt templates config.include('pyramid_chameleon') configure_content(config) configure_service(config) # configure outside connections registry = config.registry registry.db_rw = configure_db( app_config.get('ichnaea', 'db_master'), _db=_db_rw) registry.db_ro = configure_db( app_config.get('ichnaea', 'db_slave'), _db=_db_ro) registry.raven_client = raven_client = configure_raven( app_config.get('ichnaea', 'sentry_dsn'), transport='gevent', _client=_raven_client) registry.redis_client = configure_redis( app_config.get('ichnaea', 'redis_url'), _client=_redis_client) registry.stats_client = configure_stats( app_config.get('ichnaea', 'statsd_host'), _client=_stats_client) registry.geoip_db = configure_geoip( app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client, _client=_geoip_db) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW) config.add_request_method(db_rw_session, property=True) config.add_request_method(db_ro_session, property=True) # replace json renderer with custom json variant config.add_renderer('json', customjson.Renderer()) # Should we try to initialize and establish the outbound connections? if init: # pragma: no cover registry.db_ro.ping() registry.redis_client.ping() registry.stats_client.ping() return config.make_wsgi_app()
def main(argv, _db_rw=None, _redis_client=None): # pragma: no cover parser = argparse.ArgumentParser(prog=argv[0], description='Load/import cell data.') parser.add_argument('--datatype', default='ocid', help='Type of the data file, e.g. ocid') parser.add_argument('--filename', help='Path to the gzipped csv file.') args = parser.parse_args(argv[1:]) if not args.filename: parser.print_help() sys.exit(1) filename = os.path.abspath(args.filename) if not os.path.isfile(filename): print('File not found.') sys.exit(1) datatype = args.datatype if datatype not in ('cell', 'ocid'): print('Unknown data type.') sys.exit(1) configure_logging() app_config = read_config() db = configure_db(app_config.get('database', 'rw_url'), _db=_db_rw) redis_client = configure_redis(app_config.get('cache', 'cache_url'), _client=_redis_client) load_file(db, redis_client, datatype, filename)
def run_migrations_online(): db = configure_db('ddl') with db.engine.connect() as connection: context.configure(connection=connection) with connection.begin() as trans: context.run_migrations() trans.commit()
def main(argv, _db_rw=None, _redis_client=None): # pragma: no cover parser = argparse.ArgumentParser( prog=argv[0], description='Load/import cell data.') parser.add_argument('--datatype', default='ocid', help='Type of the data file, cell or ocid') parser.add_argument('--filename', help='Path to the gzipped csv file.') args = parser.parse_args(argv[1:]) if not args.filename: parser.print_help() sys.exit(1) filename = os.path.abspath(args.filename) if not os.path.isfile(filename): print('File not found.') sys.exit(1) datatype = args.datatype if datatype not in ('cell', 'ocid'): print('Unknown data type.') sys.exit(1) configure_logging() app_config = read_config() db = configure_db(app_config.get('database', 'rw_url'), _db=_db_rw) redis_client = configure_redis( app_config.get('cache', 'cache_url'), _client=_redis_client) load_file(db, redis_client, datatype, filename)
def init_worker(celery_app, app_config, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): """ Configure the passed in celery app, usually stored in :data:`ichnaea.async.app.celery_app`. Does connection, settings and queue setup. Attaches some additional functionality to the :class:`celery.Celery` instance. This is executed inside each forked worker process. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param _db_ro: Ignored, read-only database connection isn't used. """ # make config file settings available celery_app.settings = app_config.asdict() # configure outside connections celery_app.db_rw = configure_db(app_config.get('database', 'rw_url'), _db=_db_rw) celery_app.raven_client = raven_client = configure_raven( app_config.get('sentry', 'dsn'), transport='threaded', _client=_raven_client) celery_app.redis_client = redis_client = configure_redis( app_config.get('cache', 'cache_url'), _client=_redis_client) celery_app.stats_client = configure_stats(app_config, _client=_stats_client) celery_app.geoip_db = configure_geoip(app_config.get('geoip', 'db_path'), raven_client=raven_client, _client=_geoip_db) # configure data / export queues celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES]) celery_app.data_queues = data_queues = configure_data(redis_client) for queue in data_queues.values(): if queue.monitor_name: all_queues.add(queue.monitor_name) celery_app.export_queues = configure_export(redis_client, app_config) for queue in celery_app.export_queues.values(): if queue.monitor_name: all_queues.add(queue.monitor_name)
def init_worker( celery_app, app_config, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None, ): """ Configure the passed in celery app, usually stored in :data:`ichnaea.async.app.celery_app`. Does connection, settings and queue setup. Attaches some additional functionality to the :class:`celery.Celery` instance. This is executed inside each forked worker process. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param _db_ro: Ignored, read-only database connection isn't used. """ # make config file settings available celery_app.settings = app_config.asdict() # configure outside connections celery_app.db_rw = configure_db(app_config.get("ichnaea", "db_master"), _db=_db_rw) celery_app.raven_client = raven_client = configure_raven( app_config.get("ichnaea", "sentry_dsn"), transport="threaded", _client=_raven_client ) celery_app.redis_client = redis_client = configure_redis( app_config.get("ichnaea", "redis_url"), _client=_redis_client ) celery_app.stats_client = configure_stats(app_config.get("ichnaea", "statsd_host"), _client=_stats_client) celery_app.geoip_db = configure_geoip( app_config.get("ichnaea", "geoip_db_path"), raven_client=raven_client, _client=_geoip_db ) # configure data / export queues celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES]) celery_app.data_queues = data_queues = configure_data(redis_client) for queue in data_queues.values(): if queue.monitor_name: all_queues.add(queue.monitor_name) celery_app.export_queues = configure_export(redis_client, app_config) for queue in celery_app.export_queues.values(): if queue.monitor_name: all_queues.add(queue.monitor_name)
def run_migrations_online(): # Create a database connection using SQLALCHEMY_URL defined in the # environment. db_url = get_sqlalchemy_url() db = configure_db(uri=db_url, pool=False) with db.engine.connect() as connection: context.configure(connection=connection) with context.begin_transaction(): context.run_migrations()
def session2(self, database): # Create a second independent database session. db = configure_db('ro') session = db.session() try: yield session session.rollback() finally: session.close() db.close()
def init_worker(celery_app, app_config, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): # currently neither a db_ro nor geoip_db are set up # make config file settings available celery_app.settings = app_config.asdict() # configure data / export queues celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES]) celery_app.data_queues = data_queues = { 'cell_area_update': 'update_cell_lac', } for value in data_queues.values(): all_queues.add(value) celery_app.export_queues = export_queues = {} for section_name in app_config.sections(): if section_name.startswith('export:'): section = app_config.get_map(section_name) name = section_name.split(':')[1] queue_name = EXPORT_QUEUE_PREFIX + name export_queues[name] = { 'redis_key': queue_name, } all_queues.add(queue_name) for key, value in section.items(): if key == 'batch': export_queues[name][key] = int(value) else: export_queues[name][key] = value # configure outside connections celery_app.db_rw = configure_db(app_config.get('ichnaea', 'db_master'), _db=_db_rw) celery_app.raven_client = configure_raven(app_config.get( 'ichnaea', 'sentry_dsn'), transport='threaded', _client=_raven_client) celery_app.redis_client = configure_redis(app_config.get( 'ichnaea', 'redis_url'), _client=_redis_client) celery_app.stats_client = configure_stats(app_config.get( 'ichnaea', 'statsd_host'), _client=_stats_client)
def setup_database(): # Create it if it doesn't exist try: create_db() except ProgrammingError: pass # Clean up the tables and set them up db = configure_db(uri=get_sqlalchemy_url()) cleanup_tables(db.engine) setup_tables(db.engine) db.close()
def main(argv, _db=None, _dump_file=dump_file): parser = argparse.ArgumentParser(prog=argv[0], description='Dump/export data.') parser.add_argument('--datatype', required=True, help='Type of the data file, blue, cell or wifi') parser.add_argument('--filename', required=True, help='Path to the csv.gz export file.') parser.add_argument('--lat', default=None, help='The center latitude of the desired area.') parser.add_argument('--lon', default=None, help='The center longitude of the desired area.') parser.add_argument('--radius', default=None, help='The radius of the desired area.') args = parser.parse_args(argv[1:]) if not args.filename: # pragma: no cover parser.print_help() return 1 filename = os.path.abspath(os.path.expanduser(args.filename)) if os.path.isfile(filename): # pragma: no cover print('File already exists.') return 1 datatype = args.datatype if datatype not in ('blue', 'cell', 'wifi'): # pragma: no cover print('Unknown data type.') return 1 lat, lon, radius = (None, None, None) if (args.lat is not None and args.lon is not None and args.radius is not None): lat = float(args.lat) lon = float(args.lon) radius = int(args.radius) configure_logging() db = configure_db('ro', _db=_db) with db_worker_session(db, commit=False) as session: exit_code = _dump_file(datatype, session, filename, lat=lat, lon=lon, radius=radius) return exit_code
def main(argv, _db=None, _dump_file=dump_file): parser = argparse.ArgumentParser(prog=argv[0], description="Dump/export data.") parser.add_argument("--datatype", required=True, help="Type of the data file, blue, cell or wifi") parser.add_argument("--filename", required=True, help="Path to the csv.gz export file.") parser.add_argument("--lat", default=None, help="The center latitude of the desired area.") parser.add_argument("--lon", default=None, help="The center longitude of the desired area.") parser.add_argument("--radius", default=None, help="The radius of the desired area.") args = parser.parse_args(argv[1:]) if not args.filename: parser.print_help() return 1 filename = os.path.abspath(os.path.expanduser(args.filename)) if os.path.isfile(filename): print("File already exists.") return 1 datatype = args.datatype if datatype not in ("blue", "cell", "wifi"): print("Unknown data type.") return 1 lat, lon, radius = (None, None, None) if args.lat is not None and args.lon is not None and args.radius is not None: lat = float(args.lat) lon = float(args.lon) radius = int(args.radius) configure_logging() db = configure_db("ro", _db=_db, pool=False) with db_worker_session(db, commit=False) as session: exit_code = _dump_file(datatype, session, filename, lat=lat, lon=lon, radius=radius) return exit_code
def show_api_key_details(ctx, key): """Print api key details to stdout.""" db = configure_db("rw") with db_worker_session(db) as session: row = session.query(ApiKey).filter( ApiKey.valid_key == key).one_or_none() if row: api_key = Key.from_obj(row) else: api_key = None if api_key: table = [[name, value] for name, value in api_key.as_dict().items()] print_table(table, delimiter=" : ", stream_write=click_echo_no_nl) else: click.echo(f"API key '{key}' does not exist")
def show_api_key_details(key): """Print api key details to stdout.""" db = configure_db("rw") with db_worker_session(db) as session: columns = ApiKey.__table__.columns fields = [getattr(columns, f) for f in API_KEY_COLUMN_NAMES] row = (session.execute( select(fields).where(columns.valid_key == key))).fetchone() if row is not None: key = Key(**dict(row.items())) else: key = None table = [] for field in API_KEY_COLUMN_NAMES: table.append([field, getattr(key, field, "")]) print_table(table, " : ")
def export_file(filename, tablename, _db=None, _session=None): today = util.utcnow().date() one_year_ago = today - timedelta(days=365) one_year_ago = one_year_ago.strftime('%Y-%m-%d') # this is executed in a worker process stmt = text('''\ SELECT `grid`, CAST(ROUND(DATEDIFF(CURDATE(), `modified`) / 30) AS UNSIGNED) as `num` FROM {tablename} WHERE modified >= '{modified}' LIMIT :limit OFFSET :offset '''.format(tablename=tablename, modified=one_year_ago).replace('\n', ' ')) db = configure_db('ro', _db=_db) offset = 0 limit = 200000 result_rows = 0 with util.gzip_open(filename, 'w', compresslevel=2) as fd: with db_worker_session(db, commit=False) as session: if _session is not None: # testing hook session = _session while True: result = session.execute( stmt.bindparams(limit=limit, offset=offset)) rows = result.fetchall() result.close() if not rows: break lines = [] extend = lines.extend for row in rows: lat, lon = decode_datamap_grid(row.grid) extend(random_points(lat, lon, row.num)) fd.writelines(lines) result_rows += len(lines) offset += limit if not result_rows: os.remove(filename) db.close() return result_rows
def init_worker(celery_app, app_config, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): # currently neither a db_ro nor geoip_db are set up # make config file settings available celery_app.settings = app_config.asdict() # configure data / export queues celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES]) celery_app.data_queues = data_queues = { 'cell_area_update': 'update_cell_lac', } for value in data_queues.values(): all_queues.add(value) celery_app.export_queues = export_queues = {} for section_name in app_config.sections(): if section_name.startswith('export:'): section = app_config.get_map(section_name) name = section_name.split(':')[1] queue_name = EXPORT_QUEUE_PREFIX + name export_queues[name] = { 'redis_key': queue_name, } all_queues.add(queue_name) for key, value in section.items(): if key == 'batch': export_queues[name][key] = int(value) else: export_queues[name][key] = value # configure outside connections celery_app.db_rw = configure_db( app_config.get('ichnaea', 'db_master'), _db=_db_rw) celery_app.raven_client = configure_raven( app_config.get('ichnaea', 'sentry_dsn'), _client=_raven_client) celery_app.redis_client = configure_redis( app_config.get('ichnaea', 'redis_url'), _client=_redis_client) celery_app.stats_client = configure_stats( app_config.get('ichnaea', 'statsd_host'), _client=_stats_client)
def export_file(filename, tablename, _db=None, _session=None): # this is executed in a worker process stmt = text('''\ SELECT `grid`, CAST(ROUND(DATEDIFF(CURDATE(), `modified`) / 30) AS UNSIGNED) as `num` FROM {tablename} WHERE `grid` > :grid ORDER BY `grid` LIMIT :limit '''.format(tablename=tablename).replace('\n', ' ')) db = configure_db('ro', transport='sync', _db=_db) min_grid = b'' limit = 200000 result_rows = 0 with util.gzip_open(filename, 'w', compresslevel=2) as fd: with db_worker_session(db, commit=False) as session: if _session is not None: # testing hook session = _session while True: result = session.execute( stmt.bindparams(limit=limit, grid=min_grid)) rows = result.fetchall() result.close() if not rows: break lines = [] extend = lines.extend for row in rows: lat, lon = decode_datamap_grid(row.grid) extend(random_points(lat, lon, row.num)) fd.writelines(lines) result_rows += len(lines) min_grid = rows[-1].grid if not result_rows: os.remove(filename) db.close() return result_rows
def main(argv, _db_rw=None, _raven_client=None): parser = argparse.ArgumentParser(prog=argv[0], description='Initialize Ichnaea database') parser.add_argument('--alembic_ini', help='Path to the alembic migration config.') parser.add_argument('--location_ini', help='Path to the ichnaea app config.') parser.add_argument('--initdb', action='store_true', help='Initialize database') args = parser.parse_args(argv[1:]) if args.initdb: # Either use explicit config file location or fallback # on environment variable or finally file in current directory if not args.location_ini: location_ini = os.environ.get('ICHNAEA_CFG', 'location.ini') else: location_ini = args.location_ini location_ini = os.path.abspath(location_ini) location_cfg = read_config(filename=location_ini) # Either use explicit config file location or fallback # to a file in the same directory as the location.ini if not args.alembic_ini: alembic_ini = os.path.join(os.path.dirname(location_ini), 'alembic.ini') else: alembic_ini = args.alembic_ini alembic_ini = os.path.abspath(alembic_ini) alembic_cfg = Config(alembic_ini) alembic_section = alembic_cfg.get_section('alembic') db_rw = configure_db(alembic_section['sqlalchemy.url'], _db=_db_rw) configure_raven(location_cfg.get('sentry', 'dsn'), transport='sync', _client=_raven_client) engine = db_rw.engine create_schema(engine, alembic_cfg, location_cfg) else: parser.print_help()
def main(argv, _db_rw=None, _raven_client=None): parser = argparse.ArgumentParser( prog=argv[0], description='Initialize Ichnaea database') parser.add_argument('--alembic_ini', help='Path to the alembic migration config.') parser.add_argument('--location_ini', help='Path to the ichnaea app config.') parser.add_argument('--initdb', action='store_true', help='Initialize database') args = parser.parse_args(argv[1:]) if args.initdb: # Either use explicit config file location or fallback # on environment variable or finally file in current directory if not args.location_ini: location_ini = os.environ.get('ICHNAEA_CFG', 'location.ini') else: location_ini = args.location_ini location_ini = os.path.abspath(location_ini) location_cfg = read_config(filename=location_ini) # Either use explicit config file location or fallback # to a file in the same directory as the location.ini if not args.alembic_ini: alembic_ini = os.path.join( os.path.dirname(location_ini), 'alembic.ini') else: alembic_ini = args.alembic_ini alembic_ini = os.path.abspath(alembic_ini) alembic_cfg = Config(alembic_ini) alembic_section = alembic_cfg.get_section('alembic') db_rw = configure_db( alembic_section['sqlalchemy.url'], _db=_db_rw) configure_raven( location_cfg.get('sentry', 'dsn'), transport='sync', _client=_raven_client) engine = db_rw.engine create_schema(engine, alembic_cfg, location_cfg) else: parser.print_help()
def export_file(filename, tablename, _db=None, _session=None): # this is executed in a worker process stmt = text("""\ SELECT `grid`, CAST(ROUND(DATEDIFF(CURDATE(), `modified`) / 30) AS UNSIGNED) as `num` FROM {tablename} WHERE `grid` > :grid ORDER BY `grid` LIMIT :limit """.format(tablename=tablename).replace("\n", " ")) db = configure_db("ro", _db=_db, pool=False) min_grid = b"" limit = 200000 result_rows = 0 with util.gzip_open(filename, "w", compresslevel=2) as fd: with db_worker_session(db, commit=False) as session: if _session is not None: # testing hook session = _session while True: result = session.execute( stmt.bindparams(limit=limit, grid=min_grid)) rows = result.fetchall() result.close() if not rows: break lines = [] extend = lines.extend for row in rows: lat, lon = decode_datamap_grid(row.grid) extend(random_points(lat, lon, row.num)) fd.writelines(lines) result_rows += len(lines) min_grid = rows[-1].grid if not result_rows: os.remove(filename) db.close() return result_rows
def main(argv, _db=None, _dump_file=dump_file): parser = argparse.ArgumentParser( prog=argv[0], description='Dump/export data.') parser.add_argument('--datatype', required=True, help='Type of the data file, blue, cell or wifi') parser.add_argument('--filename', required=True, help='Path to the csv.gz export file.') parser.add_argument('--lat', default=None, help='The center latitude of the desired area.') parser.add_argument('--lon', default=None, help='The center longitude of the desired area.') parser.add_argument('--radius', default=None, help='The radius of the desired area.') args = parser.parse_args(argv[1:]) if not args.filename: # pragma: no cover parser.print_help() return 1 filename = os.path.abspath(os.path.expanduser(args.filename)) if os.path.isfile(filename): # pragma: no cover print('File already exists.') return 1 datatype = args.datatype if datatype not in ('blue', 'cell', 'wifi'): # pragma: no cover print('Unknown data type.') return 1 lat, lon, radius = (None, None, None) if (args.lat is not None and args.lon is not None and args.radius is not None): lat = float(args.lat) lon = float(args.lon) radius = int(args.radius) configure_logging() db = configure_db('ro', transport='sync', _db=_db) with db_worker_session(db, commit=False) as session: exit_code = _dump_file( datatype, session, filename, lat=lat, lon=lon, radius=radius) return exit_code
def create_api_key(key): """Create a new api key.""" key = key or str(uuid.uuid4()) db = configure_db("rw") with db_worker_session(db) as session: try: session.execute( insert(ApiKey.__table__).values( valid_key=key, allow_fallback=False, allow_locate=True, allow_region=True, store_sample_locate=100, store_sample_submit=100, )) print("Created API key: %r" % key) except IntegrityError: print("API key %r exists" % key)
def export_file(db_url, filename, tablename, _db_rw=None, _session=None): # this is executed in a worker process stmt = text('''\ SELECT `grid`, CAST(ROUND(DATEDIFF(CURDATE(), `modified`) / 30) AS UNSIGNED) as `num` FROM {tablename} LIMIT :limit OFFSET :offset '''.format(tablename=tablename).replace('\n', ' ')) db = configure_db(db_url, _db=_db_rw) offset = 0 limit = 200000 result_rows = 0 with util.gzip_open(filename, 'w', compresslevel=2) as fd: with db_worker_session(db, commit=False) as session: if _session is not None: # testing hook session = _session while True: result = session.execute( stmt.bindparams(limit=limit, offset=offset)) rows = result.fetchall() result.close() if not rows: break lines = [] extend = lines.extend for row in rows: lat, lon = decode_datamap_grid(row.grid) extend(random_points(lat, lon, row.num)) fd.writelines(lines) result_rows += len(lines) offset += limit if not result_rows: os.remove(filename) db.engine.pool.dispose() return result_rows
def init_worker(celery_app, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): """ Configure the passed in celery app, usually stored in :data:`ichnaea.async.app.celery_app`. Does connection, settings and queue setup. Attaches some additional functionality to the :class:`celery.Celery` instance. This is executed inside each forked worker process. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param _db_ro: Ignored, read-only database connection isn't used. """ # configure outside connections celery_app.db_rw = configure_db( celery_app.app_config.get('database', 'rw_url'), _db=_db_rw) celery_app.raven_client = raven_client = configure_raven( celery_app.app_config.get('sentry', 'dsn'), transport='threaded', _client=_raven_client) celery_app.redis_client = redis_client = configure_redis( celery_app.app_config.get('cache', 'cache_url'), _client=_redis_client) celery_app.stats_client = configure_stats( celery_app.app_config, _client=_stats_client) celery_app.geoip_db = configure_geoip( celery_app.app_config.get('geoip', 'db_path'), raven_client=raven_client, _client=_geoip_db) # configure data queues celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES]) celery_app.data_queues = data_queues = configure_data(redis_client) all_queues = all_queues.union( set([queue.key for queue in data_queues.values() if queue.key]))
def list_api_keys(ctx): """List all api keys in db.""" show_fields = [ "valid_key", "allow_fallback", "allow_locate", "allow_region" ] db = configure_db("rw") with db_worker_session(db) as session: columns = ApiKey.__table__.columns fields = [getattr(columns, f) for f in show_fields] rows = session.execute(select(fields)).fetchall() click.echo("%d api keys." % len(rows)) if rows: # Add header row table = [show_fields] # Add rest of the rows; the columns are in the order of show_fields so we # don't have to do any re-ordering table.extend(rows) print_table(table, stream_write=click_echo_no_nl)
def broken_app(self, http_session, raven): # Create database connections to the discard port. db = configure_db("rw", uri="mysql+pymysql://none:[email protected]:9/none") # Create a broken GeoIP database. geoip_db = GeoIPNull() # Create a broken Redis client. redis_client = configure_redis("redis://127.0.0.1:9/15") app = _make_app( _db=db, _geoip_db=geoip_db, _http_session=http_session, _raven_client=raven, _redis_client=redis_client, ) yield app db.close() geoip_db.close() redis_client.close()
def init_worker(celery_app, _db=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): """ Configure the passed in celery app, usually stored in :data:`ichnaea.async.app.celery_app`. Does connection, settings and queue setup. Attaches some additional functionality to the :class:`celery.Celery` instance. This is executed inside each forked worker process. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. """ # configure outside connections celery_app.db = configure_db('rw', _db=_db) celery_app.raven_client = raven_client = configure_raven( transport='threaded', _client=_raven_client) celery_app.redis_client = redis_client = configure_redis( _client=_redis_client) celery_app.stats_client = configure_stats(_client=_stats_client) celery_app.geoip_db = configure_geoip(raven_client=raven_client, _client=_geoip_db) # configure data queues celery_app.all_queues = all_queues = set([q.name for q in TASK_QUEUES]) celery_app.data_queues = data_queues = configure_data(redis_client) all_queues = all_queues.union( set([queue.key for queue in data_queues.values() if queue.key]))
def init_worker(celery_app, app_config, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): # currently db_ro is not set up # make config file settings available celery_app.settings = app_config.asdict() # configure outside connections celery_app.db_rw = configure_db( app_config.get('ichnaea', 'db_master'), _db=_db_rw) celery_app.raven_client = raven_client = configure_raven( app_config.get('ichnaea', 'sentry_dsn'), transport='threaded', _client=_raven_client) celery_app.redis_client = redis_client = configure_redis( app_config.get('ichnaea', 'redis_url'), _client=_redis_client) celery_app.stats_client = configure_stats( app_config.get('ichnaea', 'statsd_host'), _client=_stats_client) celery_app.geoip_db = configure_geoip( app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client, _client=_geoip_db) # configure data / export queues celery_app.all_queues = all_queues = set([q.name for q in CELERY_QUEUES]) celery_app.data_queues = data_queues = configure_data(redis_client) for queue in data_queues.values(): if queue.monitor_name: all_queues.add(queue.monitor_name) celery_app.export_queues = configure_export(redis_client, app_config) for queue in celery_app.export_queues.values(): if queue.monitor_name: all_queues.add(queue.monitor_name)
def broken_app(self, http_session, raven, stats): # Create database connections to the discard port. db = configure_db(uri='mysql+pymysql://none:[email protected]:9/none') # Create a broken GeoIP database. geoip_db = GeoIPNull() # Create a broken Redis client. redis_client = configure_redis('redis://127.0.0.1:9/15') app = _make_app( _db=db, _geoip_db=geoip_db, _http_session=http_session, _raven_client=raven, _redis_client=redis_client, _stats_client=stats, ) yield app db.close() geoip_db.close() redis_client.close()
def create_api_key(ctx, maxreq, key): """Create a new api key. If KEY is not specified, it uses a uuid4. """ key = key or str(uuid.uuid4()) db = configure_db("rw") with db_worker_session(db) as session: try: session.execute( insert(ApiKey.__table__).values( valid_key=key, maxreq=maxreq, allow_fallback=False, allow_locate=True, allow_region=True, store_sample_locate=100, store_sample_submit=100, )) click.echo("Created API key: %r" % key) except IntegrityError: click.echo("API key %r exists" % key)
def main(app_config, ping_connections=False, _db_rw=None, _db_ro=None, _geoip_db=None, _http_session=None, _raven_client=None, _redis_client=None, _stats_client=None, _country_searcher=None, _position_searcher=None): """ Configure the web app stored in :data:`ichnaea.webapp.app._APP`. Does connection, logging and view config setup. Attaches some additional functionality to the :class:`pyramid.registry.Registry` instance. At startup ping all outbound connections like the database once, to ensure they are actually up and responding. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param app_config: The parsed application ini. :type app_config: :class:`ichnaea.config.Config` :param ping_connections: If True, ping and test outside connections. :type ping_connections: bool :returns: A configured WSGI app, the result of calling :meth:`pyramid.config.Configurator.make_wsgi_app`. """ configure_logging() # make config file settings available config = Configurator(settings=app_config.asdict()) # add support for pt templates config.include('pyramid_chameleon') # add a config setting to skip logging for some views config.registry.skip_logging = set() configure_api(config) configure_content(config) configure_monitor(config) # configure outside connections registry = config.registry registry.db_rw = configure_db( app_config.get('database', 'rw_url'), _db=_db_rw) registry.db_ro = configure_db( app_config.get('database', 'ro_url'), _db=_db_ro) registry.raven_client = raven_client = configure_raven( app_config.get('sentry', 'dsn'), transport='gevent', _client=_raven_client) registry.redis_client = redis_client = configure_redis( app_config.get('cache', 'cache_url'), _client=_redis_client) registry.stats_client = stats_client = configure_stats( app_config, _client=_stats_client) registry.http_session = configure_http_session(_session=_http_session) registry.geoip_db = geoip_db = configure_geoip( app_config.get('geoip', 'db_path'), raven_client=raven_client, _client=_geoip_db) for name, func, default in (('country_searcher', configure_country_searcher, _country_searcher), ('position_searcher', configure_position_searcher, _position_searcher)): searcher = func(app_config, geoip_db=geoip_db, raven_client=raven_client, redis_client=redis_client, stats_client=stats_client, _searcher=default) setattr(registry, name, searcher) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW) config.add_request_method(db_rw_session, property=True) config.add_request_method(db_ro_session, property=True) # Add special JSON renderer with nicer float representation config.add_renderer('floatjson', floatjson.FloatJSONRenderer()) # freeze skip logging set config.registry.skip_logging = frozenset(config.registry.skip_logging) # Should we try to initialize and establish the outbound connections? if ping_connections: # pragma: no cover registry.db_ro.ping() registry.redis_client.ping() return config.make_wsgi_app()
def _make_db(uri=SQLURI): return configure_db(uri)
def main(global_config, app_config, init=False, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): configure_logging() # make config file settings available config = Configurator(settings=app_config.asdict()) # add support for pt templates config.include('pyramid_chameleon') configure_content(config) configure_service(config) # configure outside connections registry = config.registry registry.db_rw = configure_db(app_config.get('ichnaea', 'db_master'), _db=_db_rw) registry.db_ro = configure_db(app_config.get('ichnaea', 'db_slave'), _db=_db_ro) registry.raven_client = raven_client = configure_raven( app_config.get('ichnaea', 'sentry_dsn'), transport='gevent', _client=_raven_client) registry.redis_client = configure_redis(app_config.get( 'ichnaea', 'redis_url'), _client=_redis_client) registry.stats_client = configure_stats(app_config.get( 'ichnaea', 'statsd_host'), _client=_stats_client) registry.geoip_db = configure_geoip(app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client, _client=_geoip_db) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW) config.add_request_method(db_rw_session, property=True) config.add_request_method(db_ro_session, property=True) # replace json renderer with custom json variant config.add_renderer('json', customjson.Renderer()) # Should we try to initialize and establish the outbound connections? if init: # pragma: no cover registry.db_ro.ping() registry.redis_client.ping() registry.stats_client.ping() return config.make_wsgi_app()
def main(ping_connections=False, _db=None, _geoip_db=None, _http_session=None, _raven_client=None, _redis_client=None, _stats_client=None, _position_searcher=None, _region_searcher=None): """ Configure the web app stored in :data:`ichnaea.webapp.app._APP`. Does connection, logging and view config setup. Attaches some additional functionality to the :class:`pyramid.registry.Registry` instance. At startup ping all outbound connections like the database once, to ensure they are actually up and responding. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param ping_connections: If True, ping and test outside connections. :type ping_connections: bool :returns: A configured WSGI app, the result of calling :meth:`pyramid.config.Configurator.make_wsgi_app`. """ configure_logging() config = Configurator() # add support for pt templates config.include('pyramid_chameleon') # add a config setting to skip logging for some views config.registry.skip_logging = set() configure_api(config) configure_content(config) configure_monitor(config) # configure outside connections registry = config.registry registry.db = configure_db('ro', _db=_db) registry.raven_client = raven_client = configure_raven( transport='gevent', _client=_raven_client) registry.redis_client = redis_client = configure_redis( _client=_redis_client) registry.stats_client = stats_client = configure_stats( _client=_stats_client) registry.http_session = configure_http_session(_session=_http_session) registry.geoip_db = geoip_db = configure_geoip( raven_client=raven_client, _client=_geoip_db) # Needs to be the exact same as the *_incoming entries in async.config. registry.data_queues = data_queues = { 'update_incoming': DataQueue('update_incoming', redis_client, batch=100, compress=True), } for name, func, default in (('position_searcher', configure_position_searcher, _position_searcher), ('region_searcher', configure_region_searcher, _region_searcher)): searcher = func(geoip_db=geoip_db, raven_client=raven_client, redis_client=redis_client, stats_client=stats_client, data_queues=data_queues, _searcher=default) setattr(registry, name, searcher) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW) config.add_request_method(db_session, property=True) # freeze skip logging set config.registry.skip_logging = frozenset(config.registry.skip_logging) # Should we try to initialize and establish the outbound connections? if ping_connections: # pragma: no cover registry.db.ping() registry.redis_client.ping() return config.make_wsgi_app()
def export_to_csv(filename, csv_dir, tablename, row_limit=None, file_limit=None): """ Export a datamap table to a CSV file. :param filename: An output file ending in .csv :param csv_dir: The output directory :param tablename: The name of the datamap table to export :param row_limit: The number of rows to fetch at a time :param file_limit: The number of output rows before rotating files :return: A tuple (rows exported, files created) Each database row is turned into 0 to 6 similar CSV rows by random_points(), based on how recently they were recorded. If file_limit is not reached, the output file will the filename. If file_limit is reached, the output files will have a serial number and be based on the filename. For example, "map.csv" will become "map_0001.csv", "map_0002.csv", etc. """ stmt = text( """\ SELECT `grid`, CAST(ROUND(DATEDIFF(CURDATE(), `modified`) / 30) AS UNSIGNED) as `num` FROM {tablename} WHERE `grid` > :grid ORDER BY `grid` LIMIT :limit """.format( tablename=tablename ).replace( "\n", " " ) ) db = configure_db("ro", pool=False) min_grid = b"" row_limit = row_limit or 200_000 file_limit = file_limit or 10_000_000 result_rows = 0 file_path = os.path.join(csv_dir, filename) fd = open(file_path, "w") file_count = 1 file_rows = 0 orig_filename = filename orig_file_path = file_path assert filename.endswith(".csv") try: with db_worker_session(db, commit=False) as session: while True: result = session.execute( stmt.bindparams(limit=row_limit, grid=min_grid) ) rows = result.fetchall() result.close() if not rows: break lines = [] extend = lines.extend for row in rows: lat, lon = decode_datamap_grid(row.grid) extend(random_points(lat, lon, row.num)) fd.writelines(lines) result_rows += len(lines) # Rotate the file when needed file_rows += len(lines) if result_rows >= file_limit: fd.close() file_count += 1 file_rows = 0 filename = "sub" + orig_filename.replace( ".csv", f"_{file_count:04}.csv" ) file_path = os.path.join(csv_dir, filename) fd = open(file_path, "w") min_grid = rows[-1].grid finally: fd.close() if not file_rows: os.remove(file_path) file_count -= 1 if file_count > 1: # Rename first file to serial CSV format filename = "sub" + orig_filename.replace(".csv", "_0001.csv") file_path = os.path.join(csv_dir, filename) os.rename(orig_file_path, file_path) db.close() return result_rows, file_count
def db(database): db = configure_db(uri=get_sqlalchemy_url()) yield db db.close()
def setup_database(): db = configure_db('ddl') cleanup_tables(db.engine) setup_tables(db.engine) db.close()
def db(database): db = configure_db('rw') yield db db.close()
def main(argv, _db_rw=None, _raven_client=None, _stats_client=None): # run for example via: # bin/location_map --create --upload --datamaps=/path/to/datamaps/ \ # --output=ichnaea/content/static/tiles/ parser = argparse.ArgumentParser( prog=argv[0], description='Generate and upload datamap tiles.') parser.add_argument('--create', action='store_true', help='Create tiles.') parser.add_argument('--upload', action='store_true', help='Upload tiles to S3.') parser.add_argument('--concurrency', default=2, help='How many concurrent render processes to use?') parser.add_argument('--datamaps', help='Directory of the datamaps tools.') parser.add_argument('--output', help='Optional directory for local tile output.') args = parser.parse_args(argv[1:]) if args.create: conf = read_config() db = configure_db( conf.get('database', 'rw_url'), _db=_db_rw) raven_client = configure_raven( conf.get('sentry', 'dsn'), transport='sync', _client=_raven_client) stats_client = configure_stats(conf, _client=_stats_client) bucketname = conf.get('assets', 'bucket').strip('/') upload = False if args.upload: # pragma: no cover upload = bool(args.upload) concurrency = 2 if args.concurrency: concurrency = int(args.concurrency) datamaps = '' if args.datamaps: datamaps = os.path.abspath(args.datamaps) output = None if args.output: output = os.path.abspath(args.output) try: with stats_client.timed('datamaps', tags=['func:main']): generate(db, bucketname, raven_client, stats_client, upload=upload, concurrency=concurrency, datamaps=datamaps, output=output) except Exception: # pragma: no cover raven_client.captureException() raise else: # pragma: no cover parser.print_help()
def main(app_config, ping_connections=False, _db_rw=None, _db_ro=None, _geoip_db=None, _raven_client=None, _redis_client=None, _stats_client=None): """ Configure the web app stored in :data:`ichnaea.webapp.app._APP`. Does connection, logging and view config setup. Attaches some additional functionality to the :class:`pyramid.registry.Registry` instance. At startup ping all outbound connections like the database once, to ensure they are actually up and responding. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param app_config: The parsed application ini. :type app_config: :class:`ichnaea.config.Config` :param ping_connections: If True, ping and test outside connections. :type ping_connections: bool :returns: A configured WSGI app, the result of calling :meth:`pyramid.config.Configurator.make_wsgi_app`. """ configure_logging() # make config file settings available config = Configurator(settings=app_config.asdict()) # add support for pt templates config.include('pyramid_chameleon') configure_api(config) configure_content(config) configure_monitor(config) # configure outside connections registry = config.registry registry.db_rw = configure_db( app_config.get('ichnaea', 'db_master'), _db=_db_rw) registry.db_ro = configure_db( app_config.get('ichnaea', 'db_slave'), _db=_db_ro) registry.raven_client = raven_client = configure_raven( app_config.get('ichnaea', 'sentry_dsn'), transport='gevent', _client=_raven_client) registry.redis_client = configure_redis( app_config.get('ichnaea', 'redis_url'), _client=_redis_client) registry.stats_client = configure_stats( app_config.get('ichnaea', 'statsd_host'), _client=_stats_client) registry.geoip_db = configure_geoip( app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client, _client=_geoip_db) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW) config.add_request_method(db_rw_session, property=True) config.add_request_method(db_ro_session, property=True) # replace json renderer with custom json variant config.add_renderer('json', customjson.Renderer()) # Should we try to initialize and establish the outbound connections? if ping_connections: # pragma: no cover registry.db_ro.ping() registry.redis_client.ping() registry.stats_client.ping() return config.make_wsgi_app()
def sync_db(database): db = configure_db('rw', transport='sync') yield db db.close()
def main( ping_connections=False, _db=None, _geoip_db=None, _http_session=None, _raven_client=None, _redis_client=None, _position_searcher=None, _region_searcher=None, ): """ Configure the web app stored in :data:`ichnaea.webapp.app._APP`. Does connection, logging and view config setup. Attaches some additional functionality to the :class:`pyramid.registry.Registry` instance. At startup ping all outbound connections like the database once, to ensure they are actually up and responding. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param ping_connections: If True, ping and test outside connections. :type ping_connections: bool :returns: A configured WSGI app, the result of calling :meth:`pyramid.config.Configurator.make_wsgi_app`. """ configure_logging() config = Configurator() check_config() # add support for pt templates config.include("pyramid_chameleon") # add a config setting to skip logging for some views config.registry.skip_logging = set() configure_api(config) configure_content(config) configure_monitor(config) # configure outside connections registry = config.registry registry.db = configure_db("ro", _db=_db) registry.raven_client = raven_client = configure_raven( transport="gevent", tags={"app": "webapp"}, _client=_raven_client ) registry.redis_client = redis_client = configure_redis(_client=_redis_client) configure_stats() registry.http_session = configure_http_session(_session=_http_session) registry.geoip_db = geoip_db = configure_geoip( raven_client=raven_client, _client=_geoip_db ) # Needs to be the exact same as the *_incoming entries in taskapp.config. registry.data_queues = data_queues = { "update_incoming": DataQueue( "update_incoming", redis_client, "report", batch=100, compress=True ) } for name, func, default in ( ("position_searcher", configure_position_searcher, _position_searcher), ("region_searcher", configure_region_searcher, _region_searcher), ): searcher = func( geoip_db=geoip_db, raven_client=raven_client, redis_client=redis_client, data_queues=data_queues, _searcher=default, ) setattr(registry, name, searcher) config.add_tween("ichnaea.db.db_tween_factory", under=EXCVIEW) config.add_tween("ichnaea.log.log_tween_factory", under=EXCVIEW) config.add_request_method(db_session, property=True) # freeze skip logging set config.registry.skip_logging = frozenset(config.registry.skip_logging) # Should we try to initialize and establish the outbound connections? if ping_connections: with db_worker_session(registry.db, commit=False) as session: ping_session(session) registry.redis_client.ping() return config.make_wsgi_app()
def main(app_config, ping_connections=False, _db_rw=None, _db_ro=None, _geoip_db=None, _http_session=None, _raven_client=None, _redis_client=None, _stats_client=None, _position_searcher=None, _region_searcher=None): """ Configure the web app stored in :data:`ichnaea.webapp.app._APP`. Does connection, logging and view config setup. Attaches some additional functionality to the :class:`pyramid.registry.Registry` instance. At startup ping all outbound connections like the database once, to ensure they are actually up and responding. The parameters starting with an underscore are test-only hooks to provide pre-configured connection objects. :param app_config: The parsed application ini. :type app_config: :class:`ichnaea.config.Config` :param ping_connections: If True, ping and test outside connections. :type ping_connections: bool :returns: A configured WSGI app, the result of calling :meth:`pyramid.config.Configurator.make_wsgi_app`. """ configure_logging() # make config file settings available config = Configurator(settings=app_config.asdict()) # add support for pt templates config.include('pyramid_chameleon') # add a config setting to skip logging for some views config.registry.skip_logging = set() configure_api(config) configure_content(config) configure_monitor(config) # configure outside connections registry = config.registry registry.db_rw = configure_db( app_config.get('database', 'rw_url'), _db=_db_rw) registry.db_ro = configure_db( app_config.get('database', 'ro_url'), _db=_db_ro) registry.raven_client = raven_client = configure_raven( app_config.get('sentry', 'dsn'), transport='gevent', _client=_raven_client) registry.redis_client = redis_client = configure_redis( app_config.get('cache', 'cache_url'), _client=_redis_client) registry.stats_client = stats_client = configure_stats( app_config, _client=_stats_client) registry.http_session = configure_http_session(_session=_http_session) registry.geoip_db = geoip_db = configure_geoip( app_config.get('geoip', 'db_path'), raven_client=raven_client, _client=_geoip_db) for name, func, default in (('position_searcher', configure_position_searcher, _position_searcher), ('region_searcher', configure_region_searcher, _region_searcher)): searcher = func(app_config, geoip_db=geoip_db, raven_client=raven_client, redis_client=redis_client, stats_client=stats_client, _searcher=default) setattr(registry, name, searcher) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW) config.add_request_method(db_ro_session, property=True) # Add special JSON renderer with nicer float representation config.add_renderer('floatjson', floatjson.FloatJSONRenderer()) # Add text-as-JS renderer. config.add_renderer('js', renderers.JSRenderer()) # freeze skip logging set config.registry.skip_logging = frozenset(config.registry.skip_logging) # Should we try to initialize and establish the outbound connections? if ping_connections: # pragma: no cover registry.db_ro.ping() registry.redis_client.ping() return config.make_wsgi_app()
def db_ro(database): db = configure_db(SQLURI) yield db db.close()
def setup_database(): db = configure_db(SQLURI) engine = db.engine cleanup_tables(engine) setup_tables(engine) db.close()