def main(argv, _db_rw=None, _redis_client=None): # pragma: no cover
parser = argparse.ArgumentParser(
prog=argv[0], description='Load/import cell data.')
parser.add_argument('--datatype', default='ocid',
help='Type of the data file, cell or ocid')
parser.add_argument('--filename',
help='Path to the gzipped csv file.')
args = parser.parse_args(argv[1:])
if not args.filename:
parser.print_help()
sys.exit(1)
filename = os.path.abspath(args.filename)
if not os.path.isfile(filename):
print('File not found.')
sys.exit(1)
datatype = args.datatype
if datatype not in ('cell', 'ocid'):
print('Unknown data type.')
sys.exit(1)
configure_logging()
app_config = read_config()
db = configure_db(app_config.get('database', 'rw_url'), _db=_db_rw)
redis_client = configure_redis(
app_config.get('cache', 'cache_url'), _client=_redis_client)
load_file(db, redis_client, datatype, filename)
def main(argv, _db_rw=None, _redis_client=None): # pragma: no cover
parser = argparse.ArgumentParser(prog=argv[0],
description='Load/import cell data.')
parser.add_argument('--datatype',
default='ocid',
help='Type of the data file, e.g. ocid')
parser.add_argument('--filename', help='Path to the gzipped csv file.')
args = parser.parse_args(argv[1:])
if not args.filename:
parser.print_help()
sys.exit(1)
filename = os.path.abspath(args.filename)
if not os.path.isfile(filename):
print('File not found.')
sys.exit(1)
datatype = args.datatype
if datatype not in ('cell', 'ocid'):
print('Unknown data type.')
sys.exit(1)
configure_logging()
app_config = read_config()
db = configure_db(app_config.get('database', 'rw_url'), _db=_db_rw)
redis_client = configure_redis(app_config.get('cache', 'cache_url'),
_client=_redis_client)
load_file(db, redis_client, datatype, filename)
def main(argv, _db=None):
parser = argparse.ArgumentParser(
prog=argv[0],
description=(
"Import from public cell data into a local dev environment. "
"See https://location.services.mozilla.com/downloads"),
)
parser.add_argument("filename", help="Path to the csv.gz import file.")
args = parser.parse_args(argv[1:])
if not settings("local_dev_env"):
print("This script can only be run in a local dev environment.")
print("Set LOCAL_DEV_ENV=True in your environment.")
return 1
filename = os.path.abspath(os.path.expanduser(args.filename))
if not os.path.isfile(filename):
print("File %s not found." % filename)
return 1
configure_logging()
celery_app = get_eager_celery_app()
init_worker(celery_app)
cellarea_queue = celery_app.data_queues["update_cellarea"]
with db_worker_session(celery_app.db, commit=False) as session:
with gzip_open(filename, "r") as file_handle:
read_stations_from_csv(session, file_handle,
celery_app.redis_client, cellarea_queue)
return 0
def main(argv, _db=None, _dump_file=dump_file):
parser = argparse.ArgumentParser(prog=argv[0],
description='Dump/export data.')
parser.add_argument('--datatype',
required=True,
help='Type of the data file, blue, cell or wifi')
parser.add_argument('--filename',
required=True,
help='Path to the csv.gz export file.')
parser.add_argument('--lat',
default=None,
help='The center latitude of the desired area.')
parser.add_argument('--lon',
default=None,
help='The center longitude of the desired area.')
parser.add_argument('--radius',
default=None,
help='The radius of the desired area.')
args = parser.parse_args(argv[1:])
if not args.filename: # pragma: no cover
parser.print_help()
return 1
filename = os.path.abspath(os.path.expanduser(args.filename))
if os.path.isfile(filename): # pragma: no cover
print('File already exists.')
return 1
datatype = args.datatype
if datatype not in ('blue', 'cell', 'wifi'): # pragma: no cover
print('Unknown data type.')
return 1
lat, lon, radius = (None, None, None)
if (args.lat is not None and args.lon is not None
and args.radius is not None):
lat = float(args.lat)
lon = float(args.lon)
radius = int(args.radius)
configure_logging()
db = configure_db('ro', _db=_db)
with db_worker_session(db, commit=False) as session:
exit_code = _dump_file(datatype,
session,
filename,
lat=lat,
lon=lon,
radius=radius)
return exit_code
def main(global_config, app_config, init=False,
_db_rw=None, _db_ro=None, _geoip_db=None,
_raven_client=None, _redis_client=None, _stats_client=None):
configure_logging()
# make config file settings available
config = Configurator(settings=app_config.asdict())
# add support for pt templates
config.include('pyramid_chameleon')
configure_content(config)
configure_service(config)
# configure outside connections
registry = config.registry
registry.db_rw = configure_db(
app_config.get('ichnaea', 'db_master'), _db=_db_rw)
registry.db_ro = configure_db(
app_config.get('ichnaea', 'db_slave'), _db=_db_ro)
registry.raven_client = raven_client = configure_raven(
app_config.get('ichnaea', 'sentry_dsn'),
transport='gevent', _client=_raven_client)
registry.redis_client = configure_redis(
app_config.get('ichnaea', 'redis_url'), _client=_redis_client)
registry.stats_client = configure_stats(
app_config.get('ichnaea', 'statsd_host'), _client=_stats_client)
registry.geoip_db = configure_geoip(
app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client,
_client=_geoip_db)
config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
config.add_request_method(db_rw_session, property=True)
config.add_request_method(db_ro_session, property=True)
# replace json renderer with custom json variant
config.add_renderer('json', customjson.Renderer())
# Should we try to initialize and establish the outbound connections?
if init: # pragma: no cover
registry.db_ro.ping()
registry.redis_client.ping()
registry.stats_client.ping()
return config.make_wsgi_app()
def main(argv, _db=None, _dump_file=dump_file):
parser = argparse.ArgumentParser(prog=argv[0],
description="Dump/export data.")
parser.add_argument("--datatype",
required=True,
help="Type of the data file, blue, cell or wifi")
parser.add_argument("--filename",
required=True,
help="Path to the csv.gz export file.")
parser.add_argument("--lat",
default=None,
help="The center latitude of the desired area.")
parser.add_argument("--lon",
default=None,
help="The center longitude of the desired area.")
parser.add_argument("--radius",
default=None,
help="The radius of the desired area.")
args = parser.parse_args(argv[1:])
if not args.filename:
parser.print_help()
return 1
filename = os.path.abspath(os.path.expanduser(args.filename))
if os.path.isfile(filename):
print("File already exists.")
return 1
datatype = args.datatype
if datatype not in ("blue", "cell", "wifi"):
print("Unknown data type.")
return 1
lat, lon, radius = (None, None, None)
if args.lat is not None and args.lon is not None and args.radius is not None:
lat = float(args.lat)
lon = float(args.lon)
radius = int(args.radius)
configure_logging()
db = configure_db("ro", _db=_db, pool=False)
with db_worker_session(db, commit=False) as session:
exit_code = _dump_file(datatype,
session,
filename,
lat=lat,
lon=lon,
radius=radius)
return exit_code
def cmd_celerytest(ctx):
"""Run Sentry test through celery.
This creates a celery task which will send an event to Sentry
when it's executed.
"""
configure_logging()
init_worker(celery_app)
msg = "Testing Sentry configuration via celery (%s)" % str(datetime.datetime.now())
click.echo(click.style("Using message: %s" % msg, fg="green"))
click.echo(click.style("Creating celery task...", fg="green"))
sentry_test.delay(msg=msg)
def main(argv, _db=None, _dump_file=dump_file):
parser = argparse.ArgumentParser(
prog=argv[0], description='Dump/export data.')
parser.add_argument('--datatype', required=True,
help='Type of the data file, blue, cell or wifi')
parser.add_argument('--filename', required=True,
help='Path to the csv.gz export file.')
parser.add_argument('--lat', default=None,
help='The center latitude of the desired area.')
parser.add_argument('--lon', default=None,
help='The center longitude of the desired area.')
parser.add_argument('--radius', default=None,
help='The radius of the desired area.')
args = parser.parse_args(argv[1:])
if not args.filename: # pragma: no cover
parser.print_help()
return 1
filename = os.path.abspath(os.path.expanduser(args.filename))
if os.path.isfile(filename): # pragma: no cover
print('File already exists.')
return 1
datatype = args.datatype
if datatype not in ('blue', 'cell', 'wifi'): # pragma: no cover
print('Unknown data type.')
return 1
lat, lon, radius = (None, None, None)
if (args.lat is not None and
args.lon is not None and args.radius is not None):
lat = float(args.lat)
lon = float(args.lon)
radius = int(args.radius)
configure_logging()
db = configure_db('ro', transport='sync', _db=_db)
with db_worker_session(db, commit=False) as session:
exit_code = _dump_file(
datatype, session, filename, lat=lat, lon=lon, radius=radius)
return exit_code
def main(app_config, ping_connections=False,
_db_rw=None, _db_ro=None, _geoip_db=None,
_raven_client=None, _redis_client=None, _stats_client=None):
"""
Configure the web app stored in :data:`ichnaea.webapp.app._APP`.
Does connection, logging and view config setup. Attaches some
additional functionality to the :class:`pyramid.registry.Registry`
instance.
At startup ping all outbound connections like the database
once, to ensure they are actually up and responding.
The parameters starting with an underscore are test-only hooks
to provide pre-configured connection objects.
:param app_config: The parsed application ini.
:type app_config: :class:`ichnaea.config.Config`
:param ping_connections: If True, ping and test outside connections.
:type ping_connections: bool
:returns: A configured WSGI app, the result of calling
:meth:`pyramid.config.Configurator.make_wsgi_app`.
"""
configure_logging()
# make config file settings available
config = Configurator(settings=app_config.asdict())
# add support for pt templates
config.include('pyramid_chameleon')
configure_api(config)
configure_content(config)
configure_monitor(config)
# configure outside connections
registry = config.registry
registry.db_rw = configure_db(
app_config.get('ichnaea', 'db_master'), _db=_db_rw)
registry.db_ro = configure_db(
app_config.get('ichnaea', 'db_slave'), _db=_db_ro)
registry.raven_client = raven_client = configure_raven(
app_config.get('ichnaea', 'sentry_dsn'),
transport='gevent', _client=_raven_client)
registry.redis_client = configure_redis(
app_config.get('ichnaea', 'redis_url'), _client=_redis_client)
registry.stats_client = configure_stats(
app_config.get('ichnaea', 'statsd_host'), _client=_stats_client)
registry.geoip_db = configure_geoip(
app_config.get('ichnaea', 'geoip_db_path'), raven_client=raven_client,
_client=_geoip_db)
config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
config.add_request_method(db_rw_session, property=True)
config.add_request_method(db_ro_session, property=True)
# replace json renderer with custom json variant
config.add_renderer('json', customjson.Renderer())
# Should we try to initialize and establish the outbound connections?
if ping_connections: # pragma: no cover
registry.db_ro.ping()
registry.redis_client.ping()
registry.stats_client.ping()
return config.make_wsgi_app()
from alembic import context
from ichnaea.db import configure_db
from ichnaea.log import (
configure_logging,
)
def run_migrations_online():
db = configure_db('ddl')
with db.engine.connect() as connection:
context.configure(connection=connection)
with connection.begin() as trans:
context.run_migrations()
trans.commit()
configure_logging()
run_migrations_online()
def main(ping_connections=False,
_db=None, _geoip_db=None, _http_session=None,
_raven_client=None, _redis_client=None, _stats_client=None,
_position_searcher=None, _region_searcher=None):
"""
Configure the web app stored in :data:`ichnaea.webapp.app._APP`.
Does connection, logging and view config setup. Attaches some
additional functionality to the :class:`pyramid.registry.Registry`
instance.
At startup ping all outbound connections like the database
once, to ensure they are actually up and responding.
The parameters starting with an underscore are test-only hooks
to provide pre-configured connection objects.
:param ping_connections: If True, ping and test outside connections.
:type ping_connections: bool
:returns: A configured WSGI app, the result of calling
:meth:`pyramid.config.Configurator.make_wsgi_app`.
"""
configure_logging()
config = Configurator()
# add support for pt templates
config.include('pyramid_chameleon')
# add a config setting to skip logging for some views
config.registry.skip_logging = set()
configure_api(config)
configure_content(config)
configure_monitor(config)
# configure outside connections
registry = config.registry
registry.db = configure_db('ro', _db=_db)
registry.raven_client = raven_client = configure_raven(
transport='gevent', _client=_raven_client)
registry.redis_client = redis_client = configure_redis(
_client=_redis_client)
registry.stats_client = stats_client = configure_stats(
_client=_stats_client)
registry.http_session = configure_http_session(_session=_http_session)
registry.geoip_db = geoip_db = configure_geoip(
raven_client=raven_client, _client=_geoip_db)
# Needs to be the exact same as the *_incoming entries in async.config.
registry.data_queues = data_queues = {
'update_incoming': DataQueue('update_incoming', redis_client,
batch=100, compress=True),
}
for name, func, default in (('position_searcher',
configure_position_searcher,
_position_searcher),
('region_searcher',
configure_region_searcher,
_region_searcher)):
searcher = func(geoip_db=geoip_db, raven_client=raven_client,
redis_client=redis_client, stats_client=stats_client,
data_queues=data_queues, _searcher=default)
setattr(registry, name, searcher)
config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
config.add_request_method(db_session, property=True)
# freeze skip logging set
config.registry.skip_logging = frozenset(config.registry.skip_logging)
# Should we try to initialize and establish the outbound connections?
if ping_connections: # pragma: no cover
registry.db.ping()
registry.redis_client.ping()
return config.make_wsgi_app()
def main(_argv=None, _raven_client=None, _bucket_name=None):
"""
Command-line entry point.
:param _argv: Simulated sys.argv[1:] arguments for testing
:param _raven_client: override Raven client for testing
:param _bucket_name: override S3 bucket name for testing
:return: A system exit code
:rtype: int
"""
# Parse the command line
parser = get_parser()
args = parser.parse_args(_argv)
create = args.create
upload = args.upload
concurrency = args.concurrency
verbose = args.verbose
# Setup basic services
if verbose:
configure_logging(local_dev_env=True, logging_level="DEBUG")
else:
configure_logging()
raven_client = configure_raven(
transport="sync", tags={"app": "datamap"}, _client=_raven_client
)
# Check consistent output_dir, create, upload
exit_early = 0
output_dir = None
if args.output:
output_dir = os.path.abspath(args.output)
tiles_dir = os.path.join(output_dir, "tiles")
if not create and not os.path.isdir(tiles_dir):
LOG.error(
"The tiles subfolder of the --output directory should already"
" exist when calling --upload without --create, to avoid"
" deleting files from the S3 bucket.",
tiles_dir=tiles_dir,
)
exit_early = 1
else:
if create and not upload:
LOG.error(
"The --output argument is required with --create but without"
" --upload, since the temporary folder is removed at exit."
)
exit_early = 1
if upload and not create:
LOG.error(
"The --output argument is required with --upload but without"
" --create, to avoid deleting all tiles in the S3 bucket."
)
exit_early = 1
# Exit early with help message if error or nothing to do
if exit_early or not (create or upload):
parser.print_help()
return exit_early
# Determine the S3 bucket name
bucket_name = _bucket_name
if not _bucket_name:
bucket_name = settings("asset_bucket")
if bucket_name:
bucket_name = bucket_name.strip("/")
# Check that the implied credentials are authorized to use the bucket
if upload:
if not bucket_name:
LOG.error("Unable to determine upload bucket_name.")
return 1
else:
works, fail_msg = check_bucket(bucket_name)
if not works:
LOG.error(
f"Bucket {bucket_name} can not be used for uploads: {fail_msg}"
)
return 1
# Generate and upload the tiles
success = True
interrupted = False
result = {}
try:
with Timer() as timer:
if output_dir:
result = generate(
output_dir,
bucket_name,
raven_client,
create=create,
upload=upload,
concurrency=concurrency,
)
else:
with util.selfdestruct_tempdir() as temp_dir:
result = generate(
temp_dir,
bucket_name,
raven_client,
create=create,
upload=upload,
concurrency=concurrency,
)
except KeyboardInterrupt:
interrupted = True
success = False
except Exception:
raven_client.captureException()
success = False
raise
finally:
if create and upload:
task = "generation and upload"
elif create:
task = "generation"
else:
task = "upload"
if interrupted:
complete = "interrupted"
elif success:
complete = "complete"
else:
complete = "failed"
final_log = structlog.get_logger("canonical-log-line")
final_log.info(
f"Datamap tile {task} {complete} in {timer.duration_s:0.1f} seconds.",
success=success,
duration_s=timer.duration_s,
script_name="ichnaea.scripts.datamap",
create=create,
upload=upload,
concurrency=concurrency,
bucket_name=bucket_name,
**result,
)
return 0
def main(app_config, ping_connections=False,
_db_rw=None, _db_ro=None, _geoip_db=None, _http_session=None,
_raven_client=None, _redis_client=None, _stats_client=None,
_position_searcher=None, _region_searcher=None):
"""
Configure the web app stored in :data:`ichnaea.webapp.app._APP`.
Does connection, logging and view config setup. Attaches some
additional functionality to the :class:`pyramid.registry.Registry`
instance.
At startup ping all outbound connections like the database
once, to ensure they are actually up and responding.
The parameters starting with an underscore are test-only hooks
to provide pre-configured connection objects.
:param app_config: The parsed application ini.
:type app_config: :class:`ichnaea.config.Config`
:param ping_connections: If True, ping and test outside connections.
:type ping_connections: bool
:returns: A configured WSGI app, the result of calling
:meth:`pyramid.config.Configurator.make_wsgi_app`.
"""
configure_logging()
# make config file settings available
config = Configurator(settings=app_config.asdict())
# add support for pt templates
config.include('pyramid_chameleon')
# add a config setting to skip logging for some views
config.registry.skip_logging = set()
configure_api(config)
configure_content(config)
configure_monitor(config)
# configure outside connections
registry = config.registry
registry.db_rw = configure_db(
app_config.get('database', 'rw_url'), _db=_db_rw)
registry.db_ro = configure_db(
app_config.get('database', 'ro_url'), _db=_db_ro)
registry.raven_client = raven_client = configure_raven(
app_config.get('sentry', 'dsn'),
transport='gevent', _client=_raven_client)
registry.redis_client = redis_client = configure_redis(
app_config.get('cache', 'cache_url'), _client=_redis_client)
registry.stats_client = stats_client = configure_stats(
app_config, _client=_stats_client)
registry.http_session = configure_http_session(_session=_http_session)
registry.geoip_db = geoip_db = configure_geoip(
app_config.get('geoip', 'db_path'), raven_client=raven_client,
_client=_geoip_db)
for name, func, default in (('position_searcher',
configure_position_searcher,
_position_searcher),
('region_searcher',
configure_region_searcher,
_region_searcher)):
searcher = func(app_config,
geoip_db=geoip_db, raven_client=raven_client,
redis_client=redis_client, stats_client=stats_client,
_searcher=default)
setattr(registry, name, searcher)
config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
config.add_request_method(db_ro_session, property=True)
# Add special JSON renderer with nicer float representation
config.add_renderer('floatjson', floatjson.FloatJSONRenderer())
# Add text-as-JS renderer.
config.add_renderer('js', renderers.JSRenderer())
# freeze skip logging set
config.registry.skip_logging = frozenset(config.registry.skip_logging)
# Should we try to initialize and establish the outbound connections?
if ping_connections: # pragma: no cover
registry.db_ro.ping()
registry.redis_client.ping()
return config.make_wsgi_app()
def main(global_config,
app_config,
init=False,
_db_rw=None,
_db_ro=None,
_geoip_db=None,
_raven_client=None,
_redis_client=None,
_stats_client=None):
configure_logging()
# make config file settings available
config = Configurator(settings=app_config.asdict())
# add support for pt templates
config.include('pyramid_chameleon')
configure_content(config)
configure_service(config)
# configure outside connections
registry = config.registry
registry.db_rw = configure_db(app_config.get('ichnaea', 'db_master'),
_db=_db_rw)
registry.db_ro = configure_db(app_config.get('ichnaea', 'db_slave'),
_db=_db_ro)
registry.raven_client = raven_client = configure_raven(
app_config.get('ichnaea', 'sentry_dsn'),
transport='gevent',
_client=_raven_client)
registry.redis_client = configure_redis(app_config.get(
'ichnaea', 'redis_url'),
_client=_redis_client)
registry.stats_client = configure_stats(app_config.get(
'ichnaea', 'statsd_host'),
_client=_stats_client)
registry.geoip_db = configure_geoip(app_config.get('ichnaea',
'geoip_db_path'),
raven_client=raven_client,
_client=_geoip_db)
config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
config.add_request_method(db_rw_session, property=True)
config.add_request_method(db_ro_session, property=True)
# replace json renderer with custom json variant
config.add_renderer('json', customjson.Renderer())
# Should we try to initialize and establish the outbound connections?
if init: # pragma: no cover
registry.db_ro.ping()
registry.redis_client.ping()
registry.stats_client.ping()
return config.make_wsgi_app()
def main(app_config, ping_connections=False,
_db_rw=None, _db_ro=None, _geoip_db=None, _http_session=None,
_raven_client=None, _redis_client=None, _stats_client=None,
_country_searcher=None, _position_searcher=None):
"""
Configure the web app stored in :data:`ichnaea.webapp.app._APP`.
Does connection, logging and view config setup. Attaches some
additional functionality to the :class:`pyramid.registry.Registry`
instance.
At startup ping all outbound connections like the database
once, to ensure they are actually up and responding.
The parameters starting with an underscore are test-only hooks
to provide pre-configured connection objects.
:param app_config: The parsed application ini.
:type app_config: :class:`ichnaea.config.Config`
:param ping_connections: If True, ping and test outside connections.
:type ping_connections: bool
:returns: A configured WSGI app, the result of calling
:meth:`pyramid.config.Configurator.make_wsgi_app`.
"""
configure_logging()
# make config file settings available
config = Configurator(settings=app_config.asdict())
# add support for pt templates
config.include('pyramid_chameleon')
# add a config setting to skip logging for some views
config.registry.skip_logging = set()
configure_api(config)
configure_content(config)
configure_monitor(config)
# configure outside connections
registry = config.registry
registry.db_rw = configure_db(
app_config.get('database', 'rw_url'), _db=_db_rw)
registry.db_ro = configure_db(
app_config.get('database', 'ro_url'), _db=_db_ro)
registry.raven_client = raven_client = configure_raven(
app_config.get('sentry', 'dsn'),
transport='gevent', _client=_raven_client)
registry.redis_client = redis_client = configure_redis(
app_config.get('cache', 'cache_url'), _client=_redis_client)
registry.stats_client = stats_client = configure_stats(
app_config, _client=_stats_client)
registry.http_session = configure_http_session(_session=_http_session)
registry.geoip_db = geoip_db = configure_geoip(
app_config.get('geoip', 'db_path'), raven_client=raven_client,
_client=_geoip_db)
for name, func, default in (('country_searcher',
configure_country_searcher,
_country_searcher),
('position_searcher',
configure_position_searcher,
_position_searcher)):
searcher = func(app_config,
geoip_db=geoip_db, raven_client=raven_client,
redis_client=redis_client, stats_client=stats_client,
_searcher=default)
setattr(registry, name, searcher)
config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
config.add_request_method(db_rw_session, property=True)
config.add_request_method(db_ro_session, property=True)
# Add special JSON renderer with nicer float representation
config.add_renderer('floatjson', floatjson.FloatJSONRenderer())
# freeze skip logging set
config.registry.skip_logging = frozenset(config.registry.skip_logging)
# Should we try to initialize and establish the outbound connections?
if ping_connections: # pragma: no cover
registry.db_ro.ping()
registry.redis_client.ping()
return config.make_wsgi_app()
def main(
ping_connections=False,
_db=None,
_geoip_db=None,
_http_session=None,
_raven_client=None,
_redis_client=None,
_position_searcher=None,
_region_searcher=None,
):
"""
Configure the web app stored in :data:`ichnaea.webapp.app._APP`.
Does connection, logging and view config setup. Attaches some
additional functionality to the :class:`pyramid.registry.Registry`
instance.
At startup ping all outbound connections like the database
once, to ensure they are actually up and responding.
The parameters starting with an underscore are test-only hooks
to provide pre-configured connection objects.
:param ping_connections: If True, ping and test outside connections.
:type ping_connections: bool
:returns: A configured WSGI app, the result of calling
:meth:`pyramid.config.Configurator.make_wsgi_app`.
"""
configure_logging()
config = Configurator()
check_config()
# add support for pt templates
config.include("pyramid_chameleon")
# add a config setting to skip logging for some views
config.registry.skip_logging = set()
configure_api(config)
configure_content(config)
configure_monitor(config)
# configure outside connections
registry = config.registry
registry.db = configure_db("ro", _db=_db)
registry.raven_client = raven_client = configure_raven(
transport="gevent", tags={"app": "webapp"}, _client=_raven_client
)
registry.redis_client = redis_client = configure_redis(_client=_redis_client)
configure_stats()
registry.http_session = configure_http_session(_session=_http_session)
registry.geoip_db = geoip_db = configure_geoip(
raven_client=raven_client, _client=_geoip_db
)
# Needs to be the exact same as the *_incoming entries in taskapp.config.
registry.data_queues = data_queues = {
"update_incoming": DataQueue(
"update_incoming", redis_client, "report", batch=100, compress=True
)
}
for name, func, default in (
("position_searcher", configure_position_searcher, _position_searcher),
("region_searcher", configure_region_searcher, _region_searcher),
):
searcher = func(
geoip_db=geoip_db,
raven_client=raven_client,
redis_client=redis_client,
data_queues=data_queues,
_searcher=default,
)
setattr(registry, name, searcher)
config.add_tween("ichnaea.db.db_tween_factory", under=EXCVIEW)
config.add_tween("ichnaea.log.log_tween_factory", under=EXCVIEW)
config.add_request_method(db_session, property=True)
# freeze skip logging set
config.registry.skip_logging = frozenset(config.registry.skip_logging)
# Should we try to initialize and establish the outbound connections?
if ping_connections:
with db_worker_session(registry.db, commit=False) as session:
ping_session(session)
registry.redis_client.ping()
return config.make_wsgi_app()
def setup_logging_process(loglevel, logfile, format, colorize, **kwargs):
"""Called at scheduler and worker setup.
Configures logging using the same configuration as the webapp.
"""
configure_logging()
