def main(argv, _db_master=None, _heka_client=None): parser = argparse.ArgumentParser( prog=argv[0], description='Location Importer') parser.add_argument('source', help="The source file.") parser.add_argument('--userid', default=None, help='Internal userid for attribution.') args = parser.parse_args(argv[1:]) userid = None if args.userid is not None: userid = int(args.userid) conf = read_config() settings = conf.get_map('ichnaea') configure_heka(conf.filename, _heka_client=_heka_client) # configure databases incl. test override hooks if _db_master is None: db = Database(settings['db_master']) else: db = _db_master session = db.session() added = load_file(session, args.source, userid=userid) print('Added a total of %s records.' % added) session.commit() return added
def init_worker_process(signal, sender, **kw): # pragma: no cover # called automatically when `celery worker` is started # get the app in the current worker process app = app_or_default() settings = config().get_map('ichnaea') attach_database(app, settings=settings) configure_s3_backup(app, settings=settings) configure_heka()
def init_worker_process(signal, sender, **kw): # pragma: no cover # called automatically when `celery worker` is started # get the app in the current worker process app = app_or_default() conf = read_config() settings = conf.get_map('ichnaea') attach_database(app, settings=settings) attach_redis_client(app, settings=settings) configure_s3_backup(app, settings=settings) configure_ocid_import(app, settings=settings) configure_heka(conf.filename) configure_stats(settings.get('statsd_host'))
def main(argv, _db_master=None, _heka_client=None, _stats_client=None): # run for example via: # bin/location_map --create --upload --datamaps=/path/to/datamaps/ \ # --output=ichnaea/content/static/tiles/ parser = argparse.ArgumentParser( prog=argv[0], description='Generate and upload datamap tiles.') parser.add_argument('--create', action='store_true', help='Create tiles.') parser.add_argument('--upload', action='store_true', help='Upload tiles to S3.') parser.add_argument('--concurrency', default=2, help='How many concurrent render processes to use?') parser.add_argument('--datamaps', help='Directory of the datamaps tools.') parser.add_argument('--output', help='Optional directory for local tile output.') args = parser.parse_args(argv[1:]) if args.create: conf = read_config() db = Database(conf.get('ichnaea', 'db_master')) bucketname = conf.get('ichnaea', 's3_assets_bucket').strip('/') heka_client = configure_heka(conf.filename, _heka_client=_heka_client) stats_client = configure_stats( conf.get('ichnaea', 'statsd_host'), _client=_stats_client) upload = False if args.upload: upload = bool(args.upload) concurrency = 2 if args.concurrency: concurrency = int(args.concurrency) datamaps = '' if args.datamaps: datamaps = os.path.abspath(args.datamaps) output = None if args.output: output = os.path.abspath(args.output) try: with stats_client.timer("datamaps.total_time"): generate(db, bucketname, heka_client, stats_client, upload=upload, concurrency=concurrency, datamaps=datamaps, output=output) except Exception: heka_client.raven(RAVEN_ERROR) raise else: parser.print_help()
def main(argv, _db_master=None, _heka_client=None): parser = argparse.ArgumentParser( prog=argv[0], description='Initialize Ichnaea database') parser.add_argument('--alembic_ini', help='Path to the alembic migration config.') parser.add_argument('--location_ini', help='Path to the ichnaea app config.') parser.add_argument('--initdb', action='store_true', help='Initialize database') args = parser.parse_args(argv[1:]) if args.initdb: # Either use explicit config file location or fallback # on environment variable or finally file in current directory if not args.location_ini: location_ini = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini') else: location_ini = args.location_ini location_ini = os.path.abspath(location_ini) location_cfg = read_config(filename=location_ini) # Either use explicit config file location or fallback # to a file in the same directory as the ichnaea.ini if not args.alembic_ini: alembic_ini = os.path.join( os.path.dirname(location_ini), 'alembic.ini') else: alembic_ini = args.alembic_ini alembic_ini = os.path.abspath(alembic_ini) alembic_cfg = Config(alembic_ini) alembic_section = alembic_cfg.get_section('alembic') if _db_master is None: db_master = Database(alembic_section['sqlalchemy.url']) else: db_master = _db_master configure_heka(location_ini, _heka_client=_heka_client) engine = db_master.engine create_schema(engine, alembic_cfg, location_cfg) else: parser.print_help()
def main(global_config, _db_master=None, _db_slave=None, **settings): config = Configurator(settings=settings) # add support for pt templates config.include('pyramid_chameleon') settings = config.registry.settings from ichnaea.content.views import configure_content from ichnaea.service import configure_service from ichnaea.heka_logging import configure_heka configure_content(config) configure_service(config) # configure databases incl. test override hooks if _db_master is None: config.registry.db_master = Database( settings['db_master'], socket=settings.get('db_master_socket'), ) else: config.registry.db_master = _db_master if _db_slave is None: config.registry.db_slave = Database( settings['db_slave'], socket=settings.get('db_slave_socket'), create=False, ) else: config.registry.db_slave = _db_slave config.registry.geoip_db = configure_geoip(config.registry.settings) config.registry.heka_client = configure_heka(config.registry.settings) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.heka_logging.heka_tween_factory', under=EXCVIEW) config.add_request_method(db_master_session, property=True) config.add_request_method(db_slave_session, property=True) # replace json renderer with decimal json variant config.add_renderer('json', decimaljson.Renderer()) return config.make_wsgi_app()
def setup_heka(cls): # Clobber the stream with a debug version cls.heka_client = configure_heka() cls.heka_client.stream = DebugCaptureStream() cls.heka_client.encoder = NullEncoder(None)
def init_worker_process(signal, sender, **kw): # pragma: no cover # called automatically when `celery worker` is started # get the app in the current worker process app = app_or_default() attach_database(app) configure_heka()
def main(global_config, heka_config=None, init=False, _db_master=None, _db_slave=None, _heka_client=None, _redis=None, _stats_client=None, **settings): config = Configurator(settings=settings) # add support for pt templates config.include('pyramid_chameleon') settings = config.registry.settings from ichnaea.content.views import configure_content from ichnaea.service import configure_service from ichnaea.heka_logging import configure_heka from ichnaea.stats import configure_stats configure_content(config) configure_service(config) # configure databases incl. test override hooks if _db_master is None: config.registry.db_master = Database(settings['db_master']) else: config.registry.db_master = _db_master if _db_slave is None: config.registry.db_slave = Database(settings['db_slave']) else: config.registry.db_slave = _db_slave if _redis is None: config.registry.redis_client = None if 'redis_url' in settings: config.registry.redis_client = redis_client(settings['redis_url']) else: config.registry.redis_client = _redis config.registry.geoip_db = configure_geoip(config.registry.settings) if _heka_client is None: config.registry.heka_client = configure_heka(heka_config) else: config.registry.heka_client = _heka_client config.registry.stats_client = configure_stats( settings.get('statsd_host'), _client=_stats_client) config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW) config.add_tween('ichnaea.heka_logging.heka_tween_factory', under=EXCVIEW) config.add_request_method(db_master_session, property=True) config.add_request_method(db_slave_session, property=True) # replace json renderer with custom json variant config.add_renderer('json', customjson.Renderer()) # Should we try to initialize and establish the outbound connections? if init: # Test the slave DB connection with db_worker_session(config.registry.db_slave) as session: try: session.execute(select([func.now()])).first() except OperationalError: # Let the instance start, so it can recover / reconnect # to the DB later, but provide degraded service in the # meantime. pass # Test the redis connection try: config.registry.redis_client.ping() except ConnectionError: # Same as for the DB, continue with degraded service. pass return config.make_wsgi_app()