def data_queues(redis_client):
data_queues = {
'update_incoming':
DataQueue('update_incoming', redis_client, batch=100, compress=True),
'transfer_incoming':
DataQueue('transfer_incoming', redis_client, batch=100, compress=True),
}
yield data_queues
def _make_queue(self, redis, batch=0, compress=False, json=True):
return DataQueue(uuid4().hex,
redis,
"data",
batch=batch,
compress=compress,
json=json)
def data_queues(redis_client):
data_queues = {
"update_incoming": DataQueue(
"update_incoming", redis_client, batch=100, compress=True
)
}
yield data_queues
def configure_data(redis_client):
"""
Configure fixed set of data queues.
"""
data_queues = {
'update_cell':
DataQueue('update_cell', redis_client, queue_key='update_cell'), # BBB
'update_cellarea':
DataQueue('update_cellarea', redis_client,
queue_key='update_cellarea'),
'update_cellarea_ocid':
DataQueue('update_cellarea_ocid',
redis_client,
queue_key='update_cellarea_ocid'),
'update_score':
DataQueue('update_score', redis_client, queue_key='update_score'),
}
for shard_id in DataMap.shards().keys():
name = 'update_datamap_' + shard_id
data_queues[name] = DataQueue(name, redis_client, queue_key=name)
for shard_id in CellShard.shards().keys():
name = 'update_cell_' + shard_id
data_queues[name] = DataQueue(name, redis_client, queue_key=name)
for shard_id in WifiShard.shards().keys():
name = 'update_wifi_' + shard_id
data_queues[name] = DataQueue(name, redis_client, queue_key=name)
return data_queues
def configure_data(redis_client):
"""
Configure fixed set of data queues.
"""
data_queues = {
# *_incoming need to be the exact same as in webapp.config
'update_incoming': DataQueue('update_incoming', redis_client,
batch=100, compress=True),
'transfer_incoming': DataQueue('transfer_incoming', redis_client,
batch=100, compress=True),
}
for key in ('update_cellarea', ):
data_queues[key] = DataQueue(key, redis_client, batch=100, json=False)
for shard_id in BlueShard.shards().keys():
key = 'update_blue_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
for shard_id in DataMap.shards().keys():
key = 'update_datamap_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500, json=False)
for shard_id in CellShard.shards().keys():
key = 'update_cell_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
for shard_id in WifiShard.shards().keys():
key = 'update_wifi_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
return data_queues
def queue(self, queue_key, redis_client):
return DataQueue(
queue_key,
redis_client,
"exports",
batch=self.batch,
compress=False,
json=True,
)
def _make_queue(self, batch=0, compress=False, json=True):
return DataQueue(uuid4().hex,
self.redis_client,
batch=batch,
compress=compress,
json=json)
def main(
ping_connections=False,
_db=None,
_geoip_db=None,
_http_session=None,
_raven_client=None,
_redis_client=None,
_position_searcher=None,
_region_searcher=None,
):
"""
Configure the web app stored in :data:`ichnaea.webapp.app._APP`.
Does connection, logging and view config setup. Attaches some
additional functionality to the :class:`pyramid.registry.Registry`
instance.
At startup ping all outbound connections like the database
once, to ensure they are actually up and responding.
The parameters starting with an underscore are test-only hooks
to provide pre-configured connection objects.
:param ping_connections: If True, ping and test outside connections.
:type ping_connections: bool
:returns: A configured WSGI app, the result of calling
:meth:`pyramid.config.Configurator.make_wsgi_app`.
"""
configure_logging()
config = Configurator()
check_config()
# add support for pt templates
config.include("pyramid_chameleon")
# add a config setting to skip logging for some views
config.registry.skip_logging = set()
configure_api(config)
configure_content(config)
configure_monitor(config)
# configure outside connections
registry = config.registry
registry.db = configure_db("ro", _db=_db)
registry.raven_client = raven_client = configure_raven(
transport="gevent", tags={"app": "webapp"}, _client=_raven_client
)
registry.redis_client = redis_client = configure_redis(_client=_redis_client)
configure_stats()
registry.http_session = configure_http_session(_session=_http_session)
registry.geoip_db = geoip_db = configure_geoip(
raven_client=raven_client, _client=_geoip_db
)
# Needs to be the exact same as the *_incoming entries in taskapp.config.
registry.data_queues = data_queues = {
"update_incoming": DataQueue(
"update_incoming", redis_client, "report", batch=100, compress=True
)
}
for name, func, default in (
("position_searcher", configure_position_searcher, _position_searcher),
("region_searcher", configure_region_searcher, _region_searcher),
):
searcher = func(
geoip_db=geoip_db,
raven_client=raven_client,
redis_client=redis_client,
data_queues=data_queues,
_searcher=default,
)
setattr(registry, name, searcher)
config.add_tween("ichnaea.db.db_tween_factory", under=EXCVIEW)
config.add_tween("ichnaea.log.log_tween_factory", under=EXCVIEW)
config.add_request_method(db_session, property=True)
# freeze skip logging set
config.registry.skip_logging = frozenset(config.registry.skip_logging)
# Should we try to initialize and establish the outbound connections?
if ping_connections:
with db_worker_session(registry.db, commit=False) as session:
ping_session(session)
registry.redis_client.ping()
return config.make_wsgi_app()
def main(app_config,
ping_connections=False,
_db_rw=None,
_db_ro=None,
_geoip_db=None,
_http_session=None,
_raven_client=None,
_redis_client=None,
_stats_client=None,
_position_searcher=None,
_region_searcher=None):
"""
Configure the web app stored in :data:`ichnaea.webapp.app._APP`.
Does connection, logging and view config setup. Attaches some
additional functionality to the :class:`pyramid.registry.Registry`
instance.
At startup ping all outbound connections like the database
once, to ensure they are actually up and responding.
The parameters starting with an underscore are test-only hooks
to provide pre-configured connection objects.
:param app_config: The parsed application ini.
:type app_config: :class:`ichnaea.config.Config`
:param ping_connections: If True, ping and test outside connections.
:type ping_connections: bool
:returns: A configured WSGI app, the result of calling
:meth:`pyramid.config.Configurator.make_wsgi_app`.
"""
configure_logging()
# make config file settings available
config = Configurator(settings=app_config.asdict())
# add support for pt templates
config.include('pyramid_chameleon')
# add a config setting to skip logging for some views
config.registry.skip_logging = set()
configure_api(config)
configure_content(config)
configure_monitor(config)
# configure outside connections
registry = config.registry
registry.db_rw = configure_db(app_config.get('database', 'rw_url'),
_db=_db_rw)
registry.db_ro = configure_db(app_config.get('database', 'ro_url'),
_db=_db_ro)
registry.raven_client = raven_client = configure_raven(
app_config.get('sentry', 'dsn'),
transport='gevent',
_client=_raven_client)
registry.redis_client = redis_client = configure_redis(
app_config.get('cache', 'cache_url'), _client=_redis_client)
registry.stats_client = stats_client = configure_stats(
app_config, _client=_stats_client)
registry.http_session = configure_http_session(_session=_http_session)
registry.geoip_db = geoip_db = configure_geoip(app_config.get(
'geoip', 'db_path'),
raven_client=raven_client,
_client=_geoip_db)
for name, func, default in (('position_searcher',
configure_position_searcher,
_position_searcher),
('region_searcher', configure_region_searcher,
_region_searcher)):
searcher = func(geoip_db=geoip_db,
raven_client=raven_client,
redis_client=redis_client,
stats_client=stats_client,
_searcher=default)
setattr(registry, name, searcher)
# needs to be the exact same as async.config
registry.data_queues = {
'update_incoming':
DataQueue('update_incoming', redis_client, batch=100, compress=True),
}
config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
config.add_tween('ichnaea.log.log_tween_factory', under=EXCVIEW)
config.add_request_method(db_ro_session, property=True)
# Add special JSON renderer with nicer float representation
config.add_renderer('floatjson', floatjson.FloatJSONRenderer())
# freeze skip logging set
config.registry.skip_logging = frozenset(config.registry.skip_logging)
# Should we try to initialize and establish the outbound connections?
if ping_connections: # pragma: no cover
registry.db_ro.ping()
registry.redis_client.ping()
return config.make_wsgi_app()
def _data_queue(self, queue_key):
return DataQueue(queue_key,
self.redis_client,
batch=self.batch,
compress=False,
json=True)