def configure_celery(celery_app): """ Configure the celery app stored in :data:`ichnaea.async.app.celery_app`. This is executed both inside the master worker process and once in each forked worker process. This parses the application ini and reads in the :mod:`ichnaea.async.settings`. """ conf = read_config() if conf.has_section('celery'): section = conf.get_map('celery') else: # pragma: no cover # happens while building docs locally and on rtfd.org return # testing settings always_eager = bool(os.environ.get('CELERY_ALWAYS_EAGER', False)) redis_uri = os.environ.get('REDIS_URI') if always_eager and redis_uri: broker_url = redis_uri result_url = redis_uri else: # pragma: no cover broker_url = section['broker_url'] result_url = section['result_url'] celery_app.config_from_object('ichnaea.async.settings') celery_app.conf.update( BROKER_URL=broker_url, CELERY_RESULT_BACKEND=result_url, CELERY_QUEUES=CELERY_QUEUES, CELERYBEAT_SCHEDULE=celerybeat_schedule(conf), )
def configure_celery(celery_app): """ Configure the celery app stored in :data:`ichnaea.async.app.celery_app`. This is executed both inside the master worker process and once in each forked worker process. This parses the application ini and reads in the :mod:`ichnaea.async.settings`. """ conf = read_config() if conf.has_section('celery'): section = conf.get_map('celery') else: # pragma: no cover # happens while building docs locally and on rtfd.org return # testing settings always_eager = bool(os.environ.get('CELERY_ALWAYS_EAGER', False)) redis_uri = os.environ.get('REDIS_URI') if always_eager and redis_uri: broker_url = redis_uri result_url = redis_uri else: # pragma: no cover broker_url = section['broker_url'] result_url = section['result_url'] celery_app.config_from_object('ichnaea.async.settings') celery_app.conf.update( BROKER_URL=broker_url, CELERY_RESULT_BACKEND=result_url, CELERY_QUEUES=CELERY_QUEUES, CELERYBEAT_SCHEDULE=CELERYBEAT_SCHEDULE, )
def configure_celery(celery_app): conf = read_config() if conf.has_section('celery'): section = conf.get_map('celery') else: # pragma: no cover # happens while building docs locally and on rtfd.org return # testing settings always_eager = bool(os.environ.get('CELERY_ALWAYS_EAGER', False)) redis_uri = os.environ.get('REDIS_URI', 'redis://localhost:6379/1') if always_eager and redis_uri: broker_url = redis_uri result_url = redis_uri else: # pragma: no cover broker_url = section['broker_url'] result_url = section['result_url'] celery_app.config_from_object('ichnaea.async.settings') celery_app.conf.update( BROKER_URL=broker_url, CELERY_RESULT_BACKEND=result_url, CELERY_QUEUES=CELERY_QUEUES, CELERYBEAT_SCHEDULE=CELERYBEAT_SCHEDULE, )
def main(argv, _raven_client=None, _stats_client=None): # run for example via: # bin/location_map --create --upload --datamaps=/path/to/datamaps/ \ # --output=ichnaea/content/static/tiles/ parser = argparse.ArgumentParser( prog=argv[0], description='Generate and upload datamap tiles.') parser.add_argument('--create', action='store_true', help='Create tiles?') parser.add_argument('--upload', action='store_true', help='Upload tiles to S3?') parser.add_argument('--concurrency', default=2, help='How many concurrent processes to use?') parser.add_argument('--datamaps', help='Directory of the datamaps tools.') parser.add_argument('--output', help='Optional directory for output files.') args = parser.parse_args(argv[1:]) if args.create: conf = read_config() db_url = conf.get('database', 'rw_url') raven_client = configure_raven( conf.get('sentry', 'dsn'), transport='sync', _client=_raven_client) stats_client = configure_stats(conf, _client=_stats_client) bucketname = conf.get('assets', 'bucket').strip('/') upload = False if args.upload: upload = bool(args.upload) concurrency = billiard.cpu_count() if args.concurrency: concurrency = int(args.concurrency) datamaps = '' if args.datamaps: datamaps = os.path.abspath(args.datamaps) output = None if args.output: output = os.path.abspath(args.output) try: with stats_client.timed('datamaps', tags=['func:main']): generate(db_url, bucketname, raven_client, stats_client, upload=upload, concurrency=concurrency, datamaps=datamaps, output=output) except Exception: # pragma: no cover raven_client.captureException() raise else: # pragma: no cover parser.print_help()
def main(argv, _db_rw=None, _redis_client=None): # pragma: no cover parser = argparse.ArgumentParser( prog=argv[0], description='Load/import cell data.') parser.add_argument('--datatype', default='ocid', help='Type of the data file, cell or ocid') parser.add_argument('--filename', help='Path to the gzipped csv file.') args = parser.parse_args(argv[1:]) if not args.filename: parser.print_help() sys.exit(1) filename = os.path.abspath(args.filename) if not os.path.isfile(filename): print('File not found.') sys.exit(1) datatype = args.datatype if datatype not in ('cell', 'ocid'): print('Unknown data type.') sys.exit(1) configure_logging() app_config = read_config() db = configure_db(app_config.get('database', 'rw_url'), _db=_db_rw) redis_client = configure_redis( app_config.get('cache', 'cache_url'), _client=_redis_client) load_file(db, redis_client, datatype, filename)
def main(argv, _db_rw=None, _redis_client=None): # pragma: no cover parser = argparse.ArgumentParser(prog=argv[0], description='Load/import cell data.') parser.add_argument('--datatype', default='ocid', help='Type of the data file, e.g. ocid') parser.add_argument('--filename', help='Path to the gzipped csv file.') args = parser.parse_args(argv[1:]) if not args.filename: parser.print_help() sys.exit(1) filename = os.path.abspath(args.filename) if not os.path.isfile(filename): print('File not found.') sys.exit(1) datatype = args.datatype if datatype not in ('cell', 'ocid'): print('Unknown data type.') sys.exit(1) configure_logging() app_config = read_config() db = configure_db(app_config.get('database', 'rw_url'), _db=_db_rw) redis_client = configure_redis(app_config.get('cache', 'cache_url'), _client=_redis_client) load_file(db, redis_client, datatype, filename)
def configure_celery(celery_app): """ Configure the celery app stored in :data:`ichnaea.async.app.celery_app`. This is executed both inside the master worker process and once in each forked worker process. This parses the application ini and reads in the :mod:`ichnaea.async.settings`. """ # This happens at module import time and depends on a properly # set ICHNAEA_CFG. app_config = read_config() if not app_config.has_section('cache'): # pragma: no cover # Happens while building docs locally. return # Make config file settings available. celery_app.app_config = app_config celery_app.settings = app_config.asdict() # testing settings redis_uri = os.environ.get('REDIS_URI') if not redis_uri: # pragma: no cover redis_uri = app_config.get_map('cache')['cache_url'] celery_app.config_from_object('ichnaea.async.settings') celery_app.conf.update( BROKER_URL=redis_uri, CELERY_RESULT_BACKEND=redis_uri, CELERY_QUEUES=CELERY_QUEUES, )
def main(argv, _db_master=None, _heka_client=None): parser = argparse.ArgumentParser( prog=argv[0], description='Location Importer') parser.add_argument('source', help="The source file.") parser.add_argument('--userid', default=None, help='Internal userid for attribution.') args = parser.parse_args(argv[1:]) userid = None if args.userid is not None: userid = int(args.userid) conf = read_config() settings = conf.get_map('ichnaea') configure_heka(conf.filename, _heka_client=_heka_client) # configure databases incl. test override hooks if _db_master is None: db = Database(settings['db_master']) else: db = _db_master session = db.session() added = load_file(session, args.source, userid=userid) print('Added a total of %s records.' % added) session.commit() return added
def main(argv, _db_master=None, _heka_client=None, _stats_client=None): # run for example via: # bin/location_map --create --upload --datamaps=/path/to/datamaps/ \ # --output=ichnaea/content/static/tiles/ parser = argparse.ArgumentParser( prog=argv[0], description='Generate and upload datamap tiles.') parser.add_argument('--create', action='store_true', help='Create tiles.') parser.add_argument('--upload', action='store_true', help='Upload tiles to S3.') parser.add_argument('--concurrency', default=2, help='How many concurrent render processes to use?') parser.add_argument('--datamaps', help='Directory of the datamaps tools.') parser.add_argument('--output', help='Optional directory for local tile output.') args = parser.parse_args(argv[1:]) if args.create: conf = read_config() db = Database(conf.get('ichnaea', 'db_master')) bucketname = conf.get('ichnaea', 's3_assets_bucket').strip('/') heka_client = configure_heka(conf.filename, _heka_client=_heka_client) stats_client = configure_stats( conf.get('ichnaea', 'statsd_host'), _client=_stats_client) upload = False if args.upload: upload = bool(args.upload) concurrency = 2 if args.concurrency: concurrency = int(args.concurrency) datamaps = '' if args.datamaps: datamaps = os.path.abspath(args.datamaps) output = None if args.output: output = os.path.abspath(args.output) try: with stats_client.timer("datamaps.total_time"): generate(db, bucketname, heka_client, stats_client, upload=upload, concurrency=concurrency, datamaps=datamaps, output=output) except Exception: heka_client.raven(RAVEN_ERROR) raise else: parser.print_help()
def init_worker_process(signal, sender, **kw): # pragma: no cover # called automatically when `celery worker` is started # get the app in the current worker process app = app_or_default() conf = read_config() settings = conf.get_map('ichnaea') attach_database(app, settings=settings) configure_s3_backup(app, settings=settings) configure_heka(conf.filename) configure_stats(settings.get('statsd_host'))
def _statsd_host(): from ichnaea.config import read_config conf = read_config() if conf.has_section('ichnaea'): section = conf.get_map('ichnaea') else: # pragma: no cover # happens while building docs locally and on rtfd.org return return section.get('statsd_host', None)
def init_worker_process(signal, sender, **kw): # pragma: no cover """ Called automatically when `celery worker` is started. This is executed inside each forked worker process. Reads the app config and calls :func:`ichnaea.async.config.init_worker`. """ # get the app in the current worker process celery_app = app_or_default() conf = read_config() init_worker(celery_app, conf)
def application(environ, start_response): # pragma: no cover global _APP if _APP is None: conf = read_config() # Signal this call was made as part of app initialization _APP = main({}, heka_config=conf.filename, init=True, **conf.get_map('ichnaea')) if environ is None and start_response is None: # Called as part of gunicorn's post_worker_init return _APP return _APP(environ, start_response)
def test_tasks(self): tmpdir = tempfile.mkdtemp() filename = os.path.join(tmpdir, 'celerybeat-schedule') beat_app = celery_app.Beat() app_config = read_config() try: beat = beat_app.Service(app=celery_app, schedule_filename=filename) # parses the schedule as a side-effect scheduler = beat.get_scheduler() registered_tasks = set(scheduler._store['entries'].keys()) configured_tasks = set(schedule.celerybeat_schedule(app_config)) self.assertEqual(registered_tasks, configured_tasks) finally: shutil.rmtree(tmpdir)
def wsgi_app(environ, start_response): # pragma: no cover # Actual WSGI application endpoint, used on the command line via: # bin/gunicorn -c ichnaea.webapp.settings ichnaea.webapp.app:wsgi_app global _APP if _APP is None: conf = read_config() # Signal this call was made as part of app initialization _APP = main({}, conf, init=True) if environ is None and start_response is None: # Called as part of gunicorn's post_worker_init return _APP return _APP(environ, start_response)
def main(): settings = read_config().get_map('ichnaea') db = Database(settings['db_slave']) session = db.session() bad = [] offset = 0 count = 10000 results = True while results: results = False r = session.execute("select id, lat, lon, mcc, mnc, lac, cid, radio, " "total_measures from cell where " "lat is not null and lon is not null and " "mcc not in (1, 260) " "order by id limit %d offset %d" % (count, offset)) offset += count for row in r: results = True (id, lat, lon, mcc, mnc, lac, cid, radio, total_measures) = row ccs = [c.alpha2 for c in mobile_codes.mcc(str(mcc))] if not any([location_is_in_country(lat, lon, c, 1) for c in ccs]): if ccs: s = ",".join(ccs) else: continue bad.append(dict( type='Feature', properties=dict( mcc=mcc, mnc=mnc, lac=lac, cid=cid, radio=radio, total_measures=total_measures, countries=s), geometry=dict( type='Point', coordinates=[lon, lat]))) json.dump(dict(type='FeatureCollection', features=bad), sys.stdout, indent=True)
def main(argv, _db_rw=None, _raven_client=None): parser = argparse.ArgumentParser( prog=argv[0], description='Initialize Ichnaea database') parser.add_argument('--alembic_ini', help='Path to the alembic migration config.') parser.add_argument('--location_ini', help='Path to the ichnaea app config.') parser.add_argument('--initdb', action='store_true', help='Initialize database') args = parser.parse_args(argv[1:]) if args.initdb: # Either use explicit config file location or fallback # on environment variable or finally file in current directory if not args.location_ini: location_ini = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini') else: location_ini = args.location_ini location_ini = os.path.abspath(location_ini) location_cfg = read_config(filename=location_ini) # Either use explicit config file location or fallback # to a file in the same directory as the ichnaea.ini if not args.alembic_ini: alembic_ini = os.path.join( os.path.dirname(location_ini), 'alembic.ini') else: alembic_ini = args.alembic_ini alembic_ini = os.path.abspath(alembic_ini) alembic_cfg = Config(alembic_ini) alembic_section = alembic_cfg.get_section('alembic') if _db_rw is None: db_rw = Database(alembic_section['sqlalchemy.url']) else: db_rw = _db_rw configure_raven( location_cfg.get('ichnaea', 'sentry_dsn'), transport='sync', _client=_raven_client) engine = db_rw.engine create_schema(engine, alembic_cfg, location_cfg) else: parser.print_help()
def main(argv, _db_rw=None, _raven_client=None): parser = argparse.ArgumentParser(prog=argv[0], description='Initialize Ichnaea database') parser.add_argument('--alembic_ini', help='Path to the alembic migration config.') parser.add_argument('--location_ini', help='Path to the ichnaea app config.') parser.add_argument('--initdb', action='store_true', help='Initialize database') args = parser.parse_args(argv[1:]) if args.initdb: # Either use explicit config file location or fallback # on environment variable or finally file in current directory if not args.location_ini: location_ini = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini') else: location_ini = args.location_ini location_ini = os.path.abspath(location_ini) location_cfg = read_config(filename=location_ini) # Either use explicit config file location or fallback # to a file in the same directory as the ichnaea.ini if not args.alembic_ini: alembic_ini = os.path.join(os.path.dirname(location_ini), 'alembic.ini') else: alembic_ini = args.alembic_ini alembic_ini = os.path.abspath(alembic_ini) alembic_cfg = Config(alembic_ini) alembic_section = alembic_cfg.get_section('alembic') if _db_rw is None: db_rw = Database(alembic_section['sqlalchemy.url']) else: db_rw = _db_rw configure_raven(location_cfg.get('ichnaea', 'sentry_dsn'), _client=_raven_client) engine = db_rw.engine create_schema(engine, alembic_cfg, location_cfg) else: parser.print_help()
def _statsd_config(): from ichnaea.config import read_config conf = read_config() if conf.has_section('statsd'): section = conf.get_map('statsd') else: # pragma: no cover # happens while building docs locally and on rtfd.org return (None, None) url = None host = section.get('host', None) port = section.get('port', '8125') if host: url = '%s:%s' % (host, port) return ( url, section.get('metric_prefix', 'location'), )
def configure_celery(celery_app): """ Configure the celery app stored in :data:`ichnaea.async.app.celery_app`. This is executed both inside the master worker process and once in each forked worker process. This parses the application ini and reads in the :mod:`ichnaea.async.settings`. """ # This happens at module import time and depends on a properly # set ICHNAEA_CFG. app_config = read_config() if not app_config.has_section('celery'): # pragma: no cover # Happens while building docs locally. return # Make config file settings available. celery_app.app_config = app_config celery_app.settings = app_config.asdict() # testing settings always_eager = bool(os.environ.get('CELERY_ALWAYS_EAGER', False)) redis_uri = os.environ.get('REDIS_URI') if always_eager and redis_uri: broker_url = redis_uri result_url = redis_uri else: # pragma: no cover celery_section = app_config.get_map('celery') broker_url = celery_section['broker_url'] result_url = celery_section['result_url'] celery_app.config_from_object('ichnaea.async.settings') celery_app.conf.update( BROKER_URL=broker_url, CELERY_RESULT_BACKEND=result_url, CELERY_QUEUES=CELERY_QUEUES, )
def wsgi_app(environ, start_response): # pragma: no cover """ Actual WSGI application endpoint, used on the command line via: .. code-block:: bash bin/gunicorn -c ichnaea.webapp.settings ichnaea.webapp.app:wsgi_app At startup reads the app config and calls :func:`ichnaea.webapp.config.main` once to setup the web app stored in the :data:`ichnaea.webapp.app._APP` global. """ global _APP if _APP is None: conf = read_config() _APP = main(conf, ping_connections=True) if environ is None and start_response is None: # Called as part of gunicorn's post_worker_init return _APP return _APP(environ, start_response)
def _statsd_host(): from ichnaea.config import read_config conf = read_config() return conf.get_map('ichnaea').get('statsd_host', None)
def init_worker_process(signal, sender, **kw): # pragma: no cover # called automatically when `celery worker` is started # get the app in the current worker process celery_app = app_or_default() conf = read_config() init_worker(celery_app, conf)
from unittest import TestCase TEST_DIRECTORY = os.path.dirname(__file__) DATA_DIRECTORY = os.path.join(TEST_DIRECTORY, 'data') GEOIP_TEST_FILE = os.path.join(DATA_DIRECTORY, 'GeoIP2-City-Test.mmdb') GEOIP_BAD_FILE = os.path.join( DATA_DIRECTORY, 'GeoIP2-Connection-Type-Test.mmdb') SQLURI = os.environ.get('SQLURI') REDIS_URI = os.environ.get('REDIS_URI') SESSION = {} # Some test-data constants TEST_CONFIG = read_config(filename=os.path.join(DATA_DIRECTORY, 'test.ini')) GEOIP_DATA = { 'London': { 'city': True, 'region_code': 'GB', 'region_name': 'United Kingdom', 'ip': '81.2.69.192', 'latitude': 51.5142, 'longitude': -0.0931, 'radius': CITY_RADII[2643743], 'region_radius': GEOCODER.region_max_radius('GB'), 'score': 0.8, }, 'Bhutan': { 'city': False,
def main(argv, _db_rw=None, _raven_client=None, _stats_client=None): # run for example via: # bin/location_map --create --upload --datamaps=/path/to/datamaps/ \ # --output=ichnaea/content/static/tiles/ parser = argparse.ArgumentParser( prog=argv[0], description='Generate and upload datamap tiles.') parser.add_argument('--create', action='store_true', help='Create tiles.') parser.add_argument('--upload', action='store_true', help='Upload tiles to S3.') parser.add_argument('--concurrency', default=2, help='How many concurrent render processes to use?') parser.add_argument('--datamaps', help='Directory of the datamaps tools.') parser.add_argument('--output', help='Optional directory for local tile output.') args = parser.parse_args(argv[1:]) if args.create: conf = read_config() if _db_rw: db = _db_rw else: # pragma: no cover db = Database(conf.get('ichnaea', 'db_master')) bucketname = conf.get('ichnaea', 's3_assets_bucket').strip('/') raven_client = configure_raven(conf.get('ichnaea', 'sentry_dsn'), _client=_raven_client) stats_client = configure_stats(conf.get('ichnaea', 'statsd_host'), _client=_stats_client) upload = False if args.upload: # pragma: no cover upload = bool(args.upload) concurrency = 2 if args.concurrency: concurrency = int(args.concurrency) datamaps = '' if args.datamaps: datamaps = os.path.abspath(args.datamaps) output = None if args.output: output = os.path.abspath(args.output) try: with stats_client.timer("datamaps.total_time"): generate(db, bucketname, raven_client, stats_client, upload=upload, concurrency=concurrency, datamaps=datamaps, output=output) except Exception: # pragma: no cover raven_client.captureException() raise else: # pragma: no cover parser.print_help()
from unittest import TestCase TEST_DIRECTORY = os.path.dirname(__file__) DATA_DIRECTORY = os.path.join(TEST_DIRECTORY, 'data') GEOIP_TEST_FILE = os.path.join(DATA_DIRECTORY, 'GeoIP2-City-Test.mmdb') GEOIP_BAD_FILE = os.path.join(DATA_DIRECTORY, 'GeoIP2-Connection-Type-Test.mmdb') SQLURI = os.environ.get('SQLURI') REDIS_URI = os.environ.get('REDIS_URI') SESSION = {} # Some test-data constants TEST_CONFIG = read_config(filename=os.path.join(DATA_DIRECTORY, 'test.ini')) GEOIP_DATA = { 'London': { 'city': True, 'region_code': 'GB', 'region_name': 'United Kingdom', 'ip': '81.2.69.192', 'latitude': 51.5142, 'longitude': -0.0931, 'radius': CITY_RADII[2643743], 'region_radius': GEOCODER.region_max_radius('GB'), 'score': 0.8, }, 'Bhutan': { 'city': False,