def main(argv, _db_master=None, _heka_client=None):
parser = argparse.ArgumentParser(
prog=argv[0], description='Location Importer')
parser.add_argument('source', help="The source file.")
parser.add_argument('--userid', default=None,
help='Internal userid for attribution.')
args = parser.parse_args(argv[1:])
userid = None
if args.userid is not None:
userid = int(args.userid)
conf = read_config()
settings = conf.get_map('ichnaea')
configure_heka(conf.filename, _heka_client=_heka_client)
# configure databases incl. test override hooks
if _db_master is None:
db = Database(settings['db_master'])
else:
db = _db_master
session = db.session()
added = load_file(session, args.source, userid=userid)
print('Added a total of %s records.' % added)
session.commit()
return added
def init_worker_process(signal, sender, **kw): # pragma: no cover
# called automatically when `celery worker` is started
# get the app in the current worker process
app = app_or_default()
settings = config().get_map('ichnaea')
attach_database(app, settings=settings)
configure_s3_backup(app, settings=settings)
configure_heka()
def init_worker_process(signal, sender, **kw): # pragma: no cover
# called automatically when `celery worker` is started
# get the app in the current worker process
app = app_or_default()
conf = read_config()
settings = conf.get_map('ichnaea')
attach_database(app, settings=settings)
attach_redis_client(app, settings=settings)
configure_s3_backup(app, settings=settings)
configure_ocid_import(app, settings=settings)
configure_heka(conf.filename)
configure_stats(settings.get('statsd_host'))
def main(argv, _db_master=None, _heka_client=None, _stats_client=None):
# run for example via:
# bin/location_map --create --upload --datamaps=/path/to/datamaps/ \
# --output=ichnaea/content/static/tiles/
parser = argparse.ArgumentParser(
prog=argv[0], description='Generate and upload datamap tiles.')
parser.add_argument('--create', action='store_true',
help='Create tiles.')
parser.add_argument('--upload', action='store_true',
help='Upload tiles to S3.')
parser.add_argument('--concurrency', default=2,
help='How many concurrent render processes to use?')
parser.add_argument('--datamaps',
help='Directory of the datamaps tools.')
parser.add_argument('--output',
help='Optional directory for local tile output.')
args = parser.parse_args(argv[1:])
if args.create:
conf = read_config()
db = Database(conf.get('ichnaea', 'db_master'))
bucketname = conf.get('ichnaea', 's3_assets_bucket').strip('/')
heka_client = configure_heka(conf.filename, _heka_client=_heka_client)
stats_client = configure_stats(
conf.get('ichnaea', 'statsd_host'), _client=_stats_client)
upload = False
if args.upload:
upload = bool(args.upload)
concurrency = 2
if args.concurrency:
concurrency = int(args.concurrency)
datamaps = ''
if args.datamaps:
datamaps = os.path.abspath(args.datamaps)
output = None
if args.output:
output = os.path.abspath(args.output)
try:
with stats_client.timer("datamaps.total_time"):
generate(db, bucketname, heka_client, stats_client,
upload=upload,
concurrency=concurrency,
datamaps=datamaps,
output=output)
except Exception:
heka_client.raven(RAVEN_ERROR)
raise
else:
parser.print_help()
def main(argv, _db_master=None, _heka_client=None):
parser = argparse.ArgumentParser(
prog=argv[0], description='Initialize Ichnaea database')
parser.add_argument('--alembic_ini',
help='Path to the alembic migration config.')
parser.add_argument('--location_ini',
help='Path to the ichnaea app config.')
parser.add_argument('--initdb', action='store_true',
help='Initialize database')
args = parser.parse_args(argv[1:])
if args.initdb:
# Either use explicit config file location or fallback
# on environment variable or finally file in current directory
if not args.location_ini:
location_ini = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini')
else:
location_ini = args.location_ini
location_ini = os.path.abspath(location_ini)
location_cfg = read_config(filename=location_ini)
# Either use explicit config file location or fallback
# to a file in the same directory as the ichnaea.ini
if not args.alembic_ini:
alembic_ini = os.path.join(
os.path.dirname(location_ini), 'alembic.ini')
else:
alembic_ini = args.alembic_ini
alembic_ini = os.path.abspath(alembic_ini)
alembic_cfg = Config(alembic_ini)
alembic_section = alembic_cfg.get_section('alembic')
if _db_master is None:
db_master = Database(alembic_section['sqlalchemy.url'])
else:
db_master = _db_master
configure_heka(location_ini, _heka_client=_heka_client)
engine = db_master.engine
create_schema(engine, alembic_cfg, location_cfg)
else:
parser.print_help()
def main(global_config, _db_master=None, _db_slave=None, **settings):
config = Configurator(settings=settings)
# add support for pt templates
config.include('pyramid_chameleon')
settings = config.registry.settings
from ichnaea.content.views import configure_content
from ichnaea.service import configure_service
from ichnaea.heka_logging import configure_heka
configure_content(config)
configure_service(config)
# configure databases incl. test override hooks
if _db_master is None:
config.registry.db_master = Database(
settings['db_master'],
socket=settings.get('db_master_socket'),
)
else:
config.registry.db_master = _db_master
if _db_slave is None:
config.registry.db_slave = Database(
settings['db_slave'],
socket=settings.get('db_slave_socket'),
create=False,
)
else:
config.registry.db_slave = _db_slave
config.registry.geoip_db = configure_geoip(config.registry.settings)
config.registry.heka_client = configure_heka(config.registry.settings)
config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
config.add_tween('ichnaea.heka_logging.heka_tween_factory', under=EXCVIEW)
config.add_request_method(db_master_session, property=True)
config.add_request_method(db_slave_session, property=True)
# replace json renderer with decimal json variant
config.add_renderer('json', decimaljson.Renderer())
return config.make_wsgi_app()
def setup_heka(cls):
# Clobber the stream with a debug version
cls.heka_client = configure_heka()
cls.heka_client.stream = DebugCaptureStream()
cls.heka_client.encoder = NullEncoder(None)
def init_worker_process(signal, sender, **kw): # pragma: no cover
# called automatically when `celery worker` is started
# get the app in the current worker process
app = app_or_default()
attach_database(app)
configure_heka()
def init_worker_process(signal, sender, **kw): # pragma: no cover
# called automatically when `celery worker` is started
# get the app in the current worker process
app = app_or_default()
attach_database(app)
configure_heka()
def main(global_config, heka_config=None, init=False,
_db_master=None, _db_slave=None, _heka_client=None, _redis=None,
_stats_client=None, **settings):
config = Configurator(settings=settings)
# add support for pt templates
config.include('pyramid_chameleon')
settings = config.registry.settings
from ichnaea.content.views import configure_content
from ichnaea.service import configure_service
from ichnaea.heka_logging import configure_heka
from ichnaea.stats import configure_stats
configure_content(config)
configure_service(config)
# configure databases incl. test override hooks
if _db_master is None:
config.registry.db_master = Database(settings['db_master'])
else:
config.registry.db_master = _db_master
if _db_slave is None:
config.registry.db_slave = Database(settings['db_slave'])
else:
config.registry.db_slave = _db_slave
if _redis is None:
config.registry.redis_client = None
if 'redis_url' in settings:
config.registry.redis_client = redis_client(settings['redis_url'])
else:
config.registry.redis_client = _redis
config.registry.geoip_db = configure_geoip(config.registry.settings)
if _heka_client is None:
config.registry.heka_client = configure_heka(heka_config)
else:
config.registry.heka_client = _heka_client
config.registry.stats_client = configure_stats(
settings.get('statsd_host'), _client=_stats_client)
config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
config.add_tween('ichnaea.heka_logging.heka_tween_factory', under=EXCVIEW)
config.add_request_method(db_master_session, property=True)
config.add_request_method(db_slave_session, property=True)
# replace json renderer with custom json variant
config.add_renderer('json', customjson.Renderer())
# Should we try to initialize and establish the outbound connections?
if init:
# Test the slave DB connection
with db_worker_session(config.registry.db_slave) as session:
try:
session.execute(select([func.now()])).first()
except OperationalError:
# Let the instance start, so it can recover / reconnect
# to the DB later, but provide degraded service in the
# meantime.
pass
# Test the redis connection
try:
config.registry.redis_client.ping()
except ConnectionError:
# Same as for the DB, continue with degraded service.
pass
return config.make_wsgi_app()
