예제 #1
0
파일: config.py 프로젝트: ingle/ichnaea
def configure_celery(celery_app):
    """
    Configure the celery app stored in :data:`ichnaea.async.app.celery_app`.
    This is executed both inside the master worker process and once in
    each forked worker process.

    This parses the application ini and reads in the
    :mod:`ichnaea.async.settings`.
    """

    conf = read_config()
    if conf.has_section('celery'):
        section = conf.get_map('celery')
    else:  # pragma: no cover
        # happens while building docs locally and on rtfd.org
        return

    # testing settings
    always_eager = bool(os.environ.get('CELERY_ALWAYS_EAGER', False))
    redis_uri = os.environ.get('REDIS_URI')

    if always_eager and redis_uri:
        broker_url = redis_uri
        result_url = redis_uri
    else:  # pragma: no cover
        broker_url = section['broker_url']
        result_url = section['result_url']

    celery_app.config_from_object('ichnaea.async.settings')
    celery_app.conf.update(
        BROKER_URL=broker_url,
        CELERY_RESULT_BACKEND=result_url,
        CELERY_QUEUES=CELERY_QUEUES,
        CELERYBEAT_SCHEDULE=celerybeat_schedule(conf),
    )
예제 #2
0
파일: config.py 프로젝트: jockehewh/ichnaea
def configure_celery(celery_app):
    """
    Configure the celery app stored in :data:`ichnaea.async.app.celery_app`.
    This is executed both inside the master worker process and once in
    each forked worker process.

    This parses the application ini and reads in the
    :mod:`ichnaea.async.settings`.
    """

    conf = read_config()
    if conf.has_section('celery'):
        section = conf.get_map('celery')
    else:  # pragma: no cover
        # happens while building docs locally and on rtfd.org
        return

    # testing settings
    always_eager = bool(os.environ.get('CELERY_ALWAYS_EAGER', False))
    redis_uri = os.environ.get('REDIS_URI')

    if always_eager and redis_uri:
        broker_url = redis_uri
        result_url = redis_uri
    else:  # pragma: no cover
        broker_url = section['broker_url']
        result_url = section['result_url']

    celery_app.config_from_object('ichnaea.async.settings')
    celery_app.conf.update(
        BROKER_URL=broker_url,
        CELERY_RESULT_BACKEND=result_url,
        CELERY_QUEUES=CELERY_QUEUES,
        CELERYBEAT_SCHEDULE=CELERYBEAT_SCHEDULE,
    )
예제 #3
0
파일: config.py 프로젝트: awoland/ichnaea
def configure_celery(celery_app):
    conf = read_config()
    if conf.has_section('celery'):
        section = conf.get_map('celery')
    else:  # pragma: no cover
        # happens while building docs locally and on rtfd.org
        return

    # testing settings
    always_eager = bool(os.environ.get('CELERY_ALWAYS_EAGER', False))
    redis_uri = os.environ.get('REDIS_URI', 'redis://localhost:6379/1')

    if always_eager and redis_uri:
        broker_url = redis_uri
        result_url = redis_uri
    else:  # pragma: no cover
        broker_url = section['broker_url']
        result_url = section['result_url']

    celery_app.config_from_object('ichnaea.async.settings')
    celery_app.conf.update(
        BROKER_URL=broker_url,
        CELERY_RESULT_BACKEND=result_url,
        CELERY_QUEUES=CELERY_QUEUES,
        CELERYBEAT_SCHEDULE=CELERYBEAT_SCHEDULE,
    )
예제 #4
0
def main(argv, _raven_client=None, _stats_client=None):
    # run for example via:
    # bin/location_map --create --upload --datamaps=/path/to/datamaps/ \
    #   --output=ichnaea/content/static/tiles/

    parser = argparse.ArgumentParser(
        prog=argv[0], description='Generate and upload datamap tiles.')

    parser.add_argument('--create', action='store_true',
                        help='Create tiles?')
    parser.add_argument('--upload', action='store_true',
                        help='Upload tiles to S3?')
    parser.add_argument('--concurrency', default=2,
                        help='How many concurrent processes to use?')
    parser.add_argument('--datamaps',
                        help='Directory of the datamaps tools.')
    parser.add_argument('--output',
                        help='Optional directory for output files.')

    args = parser.parse_args(argv[1:])
    if args.create:
        conf = read_config()
        db_url = conf.get('database', 'rw_url')

        raven_client = configure_raven(
            conf.get('sentry', 'dsn'),
            transport='sync', _client=_raven_client)

        stats_client = configure_stats(conf, _client=_stats_client)

        bucketname = conf.get('assets', 'bucket').strip('/')

        upload = False
        if args.upload:
            upload = bool(args.upload)

        concurrency = billiard.cpu_count()
        if args.concurrency:
            concurrency = int(args.concurrency)

        datamaps = ''
        if args.datamaps:
            datamaps = os.path.abspath(args.datamaps)

        output = None
        if args.output:
            output = os.path.abspath(args.output)

        try:
            with stats_client.timed('datamaps', tags=['func:main']):
                generate(db_url, bucketname, raven_client, stats_client,
                         upload=upload,
                         concurrency=concurrency,
                         datamaps=datamaps,
                         output=output)
        except Exception:  # pragma: no cover
            raven_client.captureException()
            raise
    else:  # pragma: no cover
        parser.print_help()
예제 #5
0
파일: config.py 프로젝트: thebent/ichnaea
def configure_celery(celery_app):
    conf = read_config()
    if conf.has_section('celery'):
        section = conf.get_map('celery')
    else:  # pragma: no cover
        # happens while building docs locally and on rtfd.org
        return

    # testing settings
    always_eager = bool(os.environ.get('CELERY_ALWAYS_EAGER', False))
    redis_uri = os.environ.get('REDIS_URI', 'redis://localhost:6379/1')

    if always_eager and redis_uri:
        broker_url = redis_uri
        result_url = redis_uri
    else:  # pragma: no cover
        broker_url = section['broker_url']
        result_url = section['result_url']

    celery_app.config_from_object('ichnaea.async.settings')
    celery_app.conf.update(
        BROKER_URL=broker_url,
        CELERY_RESULT_BACKEND=result_url,
        CELERY_QUEUES=CELERY_QUEUES,
        CELERYBEAT_SCHEDULE=CELERYBEAT_SCHEDULE,
    )
예제 #6
0
파일: load.py 프로젝트: amjadm61/ichnaea
def main(argv, _db_rw=None, _redis_client=None):  # pragma: no cover
    parser = argparse.ArgumentParser(
        prog=argv[0], description='Load/import cell data.')
    parser.add_argument('--datatype', default='ocid',
                        help='Type of the data file, cell or ocid')
    parser.add_argument('--filename',
                        help='Path to the gzipped csv file.')

    args = parser.parse_args(argv[1:])
    if not args.filename:
        parser.print_help()
        sys.exit(1)

    filename = os.path.abspath(args.filename)
    if not os.path.isfile(filename):
        print('File not found.')
        sys.exit(1)

    datatype = args.datatype
    if datatype not in ('cell', 'ocid'):
        print('Unknown data type.')
        sys.exit(1)

    configure_logging()
    app_config = read_config()
    db = configure_db(app_config.get('database', 'rw_url'), _db=_db_rw)
    redis_client = configure_redis(
        app_config.get('cache', 'cache_url'), _client=_redis_client)

    load_file(db, redis_client, datatype, filename)
예제 #7
0
def main(argv, _db_rw=None, _redis_client=None):  # pragma: no cover
    parser = argparse.ArgumentParser(prog=argv[0],
                                     description='Load/import cell data.')
    parser.add_argument('--datatype',
                        default='ocid',
                        help='Type of the data file, e.g. ocid')
    parser.add_argument('--filename', help='Path to the gzipped csv file.')

    args = parser.parse_args(argv[1:])
    if not args.filename:
        parser.print_help()
        sys.exit(1)

    filename = os.path.abspath(args.filename)
    if not os.path.isfile(filename):
        print('File not found.')
        sys.exit(1)

    datatype = args.datatype
    if datatype not in ('cell', 'ocid'):
        print('Unknown data type.')
        sys.exit(1)

    configure_logging()
    app_config = read_config()
    db = configure_db(app_config.get('database', 'rw_url'), _db=_db_rw)
    redis_client = configure_redis(app_config.get('cache', 'cache_url'),
                                   _client=_redis_client)

    load_file(db, redis_client, datatype, filename)
예제 #8
0
파일: config.py 프로젝트: amjadm61/ichnaea
def configure_celery(celery_app):
    """
    Configure the celery app stored in :data:`ichnaea.async.app.celery_app`.
    This is executed both inside the master worker process and once in
    each forked worker process.

    This parses the application ini and reads in the
    :mod:`ichnaea.async.settings`.
    """

    # This happens at module import time and depends on a properly
    # set ICHNAEA_CFG.
    app_config = read_config()
    if not app_config.has_section('cache'):  # pragma: no cover
        # Happens while building docs locally.
        return

    # Make config file settings available.
    celery_app.app_config = app_config
    celery_app.settings = app_config.asdict()

    # testing settings
    redis_uri = os.environ.get('REDIS_URI')

    if not redis_uri:  # pragma: no cover
        redis_uri = app_config.get_map('cache')['cache_url']

    celery_app.config_from_object('ichnaea.async.settings')
    celery_app.conf.update(
        BROKER_URL=redis_uri,
        CELERY_RESULT_BACKEND=redis_uri,
        CELERY_QUEUES=CELERY_QUEUES,
    )
예제 #9
0
파일: importer.py 프로젝트: elkos/ichnaea
def main(argv, _db_master=None, _heka_client=None):
    parser = argparse.ArgumentParser(
        prog=argv[0], description='Location Importer')

    parser.add_argument('source', help="The source file.")
    parser.add_argument('--userid', default=None,
                        help='Internal userid for attribution.')

    args = parser.parse_args(argv[1:])
    userid = None
    if args.userid is not None:
        userid = int(args.userid)

    conf = read_config()
    settings = conf.get_map('ichnaea')
    configure_heka(conf.filename, _heka_client=_heka_client)

    # configure databases incl. test override hooks
    if _db_master is None:
        db = Database(settings['db_master'])
    else:
        db = _db_master
    session = db.session()
    added = load_file(session, args.source, userid=userid)
    print('Added a total of %s records.' % added)
    session.commit()
    return added
예제 #10
0
파일: map.py 프로젝트: elkos/ichnaea
def main(argv, _db_master=None, _heka_client=None, _stats_client=None):
    # run for example via:
    # bin/location_map --create --upload --datamaps=/path/to/datamaps/ \
    #   --output=ichnaea/content/static/tiles/

    parser = argparse.ArgumentParser(
        prog=argv[0], description='Generate and upload datamap tiles.')

    parser.add_argument('--create', action='store_true',
                        help='Create tiles.')
    parser.add_argument('--upload', action='store_true',
                        help='Upload tiles to S3.')
    parser.add_argument('--concurrency', default=2,
                        help='How many concurrent render processes to use?')
    parser.add_argument('--datamaps',
                        help='Directory of the datamaps tools.')
    parser.add_argument('--output',
                        help='Optional directory for local tile output.')

    args = parser.parse_args(argv[1:])

    if args.create:
        conf = read_config()
        db = Database(conf.get('ichnaea', 'db_master'))
        bucketname = conf.get('ichnaea', 's3_assets_bucket').strip('/')
        heka_client = configure_heka(conf.filename, _heka_client=_heka_client)
        stats_client = configure_stats(
            conf.get('ichnaea', 'statsd_host'), _client=_stats_client)

        upload = False
        if args.upload:
            upload = bool(args.upload)

        concurrency = 2
        if args.concurrency:
            concurrency = int(args.concurrency)

        datamaps = ''
        if args.datamaps:
            datamaps = os.path.abspath(args.datamaps)

        output = None
        if args.output:
            output = os.path.abspath(args.output)

        try:
            with stats_client.timer("datamaps.total_time"):
                generate(db, bucketname, heka_client, stats_client,
                         upload=upload,
                         concurrency=concurrency,
                         datamaps=datamaps,
                         output=output)
        except Exception:
            heka_client.raven(RAVEN_ERROR)
            raise
    else:
        parser.print_help()
예제 #11
0
파일: worker.py 프로젝트: elkos/ichnaea
def init_worker_process(signal, sender, **kw):  # pragma: no cover
    # called automatically when `celery worker` is started
    # get the app in the current worker process
    app = app_or_default()
    conf = read_config()
    settings = conf.get_map('ichnaea')
    attach_database(app, settings=settings)
    configure_s3_backup(app, settings=settings)
    configure_heka(conf.filename)
    configure_stats(settings.get('statsd_host'))
예제 #12
0
def _statsd_host():
    from ichnaea.config import read_config

    conf = read_config()
    if conf.has_section('ichnaea'):
        section = conf.get_map('ichnaea')
    else:  # pragma: no cover
        # happens while building docs locally and on rtfd.org
        return

    return section.get('statsd_host', None)
예제 #13
0
def init_worker_process(signal, sender, **kw):  # pragma: no cover
    """
    Called automatically when `celery worker` is started. This is executed
    inside each forked worker process.

    Reads the app config and calls :func:`ichnaea.async.config.init_worker`.
    """
    # get the app in the current worker process
    celery_app = app_or_default()
    conf = read_config()
    init_worker(celery_app, conf)
예제 #14
0
파일: app.py 프로젝트: cemoulto/ichnaea
def init_worker_process(signal, sender, **kw):  # pragma: no cover
    """
    Called automatically when `celery worker` is started. This is executed
    inside each forked worker process.

    Reads the app config and calls :func:`ichnaea.async.config.init_worker`.
    """
    # get the app in the current worker process
    celery_app = app_or_default()
    conf = read_config()
    init_worker(celery_app, conf)
예제 #15
0
파일: __init__.py 프로젝트: elkos/ichnaea
def application(environ, start_response):  # pragma: no cover
    global _APP

    if _APP is None:
        conf = read_config()
        # Signal this call was made as part of app initialization
        _APP = main({}, heka_config=conf.filename, init=True,
                    **conf.get_map('ichnaea'))
        if environ is None and start_response is None:
            # Called as part of gunicorn's post_worker_init
            return _APP

    return _APP(environ, start_response)
예제 #16
0
 def test_tasks(self):
     tmpdir = tempfile.mkdtemp()
     filename = os.path.join(tmpdir, 'celerybeat-schedule')
     beat_app = celery_app.Beat()
     app_config = read_config()
     try:
         beat = beat_app.Service(app=celery_app, schedule_filename=filename)
         # parses the schedule as a side-effect
         scheduler = beat.get_scheduler()
         registered_tasks = set(scheduler._store['entries'].keys())
         configured_tasks = set(schedule.celerybeat_schedule(app_config))
         self.assertEqual(registered_tasks, configured_tasks)
     finally:
         shutil.rmtree(tmpdir)
예제 #17
0
파일: tests.py 프로젝트: ingle/ichnaea
 def test_tasks(self):
     tmpdir = tempfile.mkdtemp()
     filename = os.path.join(tmpdir, 'celerybeat-schedule')
     beat_app = celery_app.Beat()
     app_config = read_config()
     try:
         beat = beat_app.Service(app=celery_app, schedule_filename=filename)
         # parses the schedule as a side-effect
         scheduler = beat.get_scheduler()
         registered_tasks = set(scheduler._store['entries'].keys())
         configured_tasks = set(schedule.celerybeat_schedule(app_config))
         self.assertEqual(registered_tasks, configured_tasks)
     finally:
         shutil.rmtree(tmpdir)
예제 #18
0
def wsgi_app(environ, start_response):  # pragma: no cover
    # Actual WSGI application endpoint, used on the command line via:
    # bin/gunicorn -c ichnaea.webapp.settings ichnaea.webapp.app:wsgi_app
    global _APP

    if _APP is None:
        conf = read_config()
        # Signal this call was made as part of app initialization
        _APP = main({}, conf, init=True)
        if environ is None and start_response is None:
            # Called as part of gunicorn's post_worker_init
            return _APP

    return _APP(environ, start_response)
예제 #19
0
def main():

    settings = read_config().get_map('ichnaea')
    db = Database(settings['db_slave'])
    session = db.session()

    bad = []

    offset = 0
    count = 10000
    results = True
    while results:
        results = False
        r = session.execute("select id, lat, lon, mcc, mnc, lac, cid, radio, "
                            "total_measures from cell where "
                            "lat is not null and lon is not null and "
                            "mcc not in (1, 260) "
                            "order by id limit %d offset %d" %
                            (count, offset))
        offset += count
        for row in r:
            results = True
            (id, lat, lon, mcc, mnc, lac, cid, radio, total_measures) = row
            ccs = [c.alpha2 for c in mobile_codes.mcc(str(mcc))]
            if not any([location_is_in_country(lat, lon, c, 1) for c in ccs]):
                if ccs:
                    s = ",".join(ccs)
                else:
                    continue
                bad.append(dict(
                    type='Feature',
                    properties=dict(
                        mcc=mcc,
                        mnc=mnc,
                        lac=lac,
                        cid=cid,
                        radio=radio,
                        total_measures=total_measures,
                        countries=s),
                    geometry=dict(
                        type='Point',
                        coordinates=[lon, lat])))

    json.dump(dict(type='FeatureCollection',
                   features=bad),
              sys.stdout,
              indent=True)
예제 #20
0
파일: initdb.py 프로젝트: SOFTowaha/ichnaea
def main(argv, _db_rw=None, _raven_client=None):
    parser = argparse.ArgumentParser(
        prog=argv[0], description='Initialize Ichnaea database')

    parser.add_argument('--alembic_ini',
                        help='Path to the alembic migration config.')
    parser.add_argument('--location_ini',
                        help='Path to the ichnaea app config.')
    parser.add_argument('--initdb', action='store_true',
                        help='Initialize database')

    args = parser.parse_args(argv[1:])

    if args.initdb:
        # Either use explicit config file location or fallback
        # on environment variable or finally file in current directory
        if not args.location_ini:
            location_ini = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini')
        else:
            location_ini = args.location_ini
        location_ini = os.path.abspath(location_ini)
        location_cfg = read_config(filename=location_ini)

        # Either use explicit config file location or fallback
        # to a file in the same directory as the ichnaea.ini
        if not args.alembic_ini:
            alembic_ini = os.path.join(
                os.path.dirname(location_ini), 'alembic.ini')
        else:
            alembic_ini = args.alembic_ini
        alembic_ini = os.path.abspath(alembic_ini)
        alembic_cfg = Config(alembic_ini)
        alembic_section = alembic_cfg.get_section('alembic')

        if _db_rw is None:
            db_rw = Database(alembic_section['sqlalchemy.url'])
        else:
            db_rw = _db_rw
        configure_raven(
            location_cfg.get('ichnaea', 'sentry_dsn'),
            transport='sync', _client=_raven_client)

        engine = db_rw.engine
        create_schema(engine, alembic_cfg, location_cfg)
    else:
        parser.print_help()
예제 #21
0
파일: initdb.py 프로젝트: thebent/ichnaea
def main(argv, _db_rw=None, _raven_client=None):
    parser = argparse.ArgumentParser(prog=argv[0],
                                     description='Initialize Ichnaea database')

    parser.add_argument('--alembic_ini',
                        help='Path to the alembic migration config.')
    parser.add_argument('--location_ini',
                        help='Path to the ichnaea app config.')
    parser.add_argument('--initdb',
                        action='store_true',
                        help='Initialize database')

    args = parser.parse_args(argv[1:])

    if args.initdb:
        # Either use explicit config file location or fallback
        # on environment variable or finally file in current directory
        if not args.location_ini:
            location_ini = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini')
        else:
            location_ini = args.location_ini
        location_ini = os.path.abspath(location_ini)
        location_cfg = read_config(filename=location_ini)

        # Either use explicit config file location or fallback
        # to a file in the same directory as the ichnaea.ini
        if not args.alembic_ini:
            alembic_ini = os.path.join(os.path.dirname(location_ini),
                                       'alembic.ini')
        else:
            alembic_ini = args.alembic_ini
        alembic_ini = os.path.abspath(alembic_ini)
        alembic_cfg = Config(alembic_ini)
        alembic_section = alembic_cfg.get_section('alembic')

        if _db_rw is None:
            db_rw = Database(alembic_section['sqlalchemy.url'])
        else:
            db_rw = _db_rw
        configure_raven(location_cfg.get('ichnaea', 'sentry_dsn'),
                        _client=_raven_client)

        engine = db_rw.engine
        create_schema(engine, alembic_cfg, location_cfg)
    else:
        parser.print_help()
예제 #22
0
def _statsd_config():
    from ichnaea.config import read_config

    conf = read_config()
    if conf.has_section('statsd'):
        section = conf.get_map('statsd')
    else:  # pragma: no cover
        # happens while building docs locally and on rtfd.org
        return (None, None)

    url = None
    host = section.get('host', None)
    port = section.get('port', '8125')
    if host:
        url = '%s:%s' % (host, port)

    return (
        url,
        section.get('metric_prefix', 'location'),
    )
예제 #23
0
def _statsd_config():
    from ichnaea.config import read_config

    conf = read_config()
    if conf.has_section('statsd'):
        section = conf.get_map('statsd')
    else:  # pragma: no cover
        # happens while building docs locally and on rtfd.org
        return (None, None)

    url = None
    host = section.get('host', None)
    port = section.get('port', '8125')
    if host:
        url = '%s:%s' % (host, port)

    return (
        url,
        section.get('metric_prefix', 'location'),
    )
예제 #24
0
def configure_celery(celery_app):
    """
    Configure the celery app stored in :data:`ichnaea.async.app.celery_app`.
    This is executed both inside the master worker process and once in
    each forked worker process.

    This parses the application ini and reads in the
    :mod:`ichnaea.async.settings`.
    """

    # This happens at module import time and depends on a properly
    # set ICHNAEA_CFG.
    app_config = read_config()
    if not app_config.has_section('celery'):  # pragma: no cover
        # Happens while building docs locally.
        return

    # Make config file settings available.
    celery_app.app_config = app_config
    celery_app.settings = app_config.asdict()

    # testing settings
    always_eager = bool(os.environ.get('CELERY_ALWAYS_EAGER', False))
    redis_uri = os.environ.get('REDIS_URI')

    if always_eager and redis_uri:
        broker_url = redis_uri
        result_url = redis_uri
    else:  # pragma: no cover
        celery_section = app_config.get_map('celery')
        broker_url = celery_section['broker_url']
        result_url = celery_section['result_url']

    celery_app.config_from_object('ichnaea.async.settings')
    celery_app.conf.update(
        BROKER_URL=broker_url,
        CELERY_RESULT_BACKEND=result_url,
        CELERY_QUEUES=CELERY_QUEUES,
    )
예제 #25
0
파일: app.py 프로젝트: jockehewh/ichnaea
def wsgi_app(environ, start_response):  # pragma: no cover
    """
    Actual WSGI application endpoint, used on the command line via:

    .. code-block:: bash

        bin/gunicorn -c ichnaea.webapp.settings ichnaea.webapp.app:wsgi_app

    At startup reads the app config and calls
    :func:`ichnaea.webapp.config.main` once to setup the web app stored
    in the :data:`ichnaea.webapp.app._APP` global.
    """
    global _APP

    if _APP is None:
        conf = read_config()
        _APP = main(conf, ping_connections=True)
        if environ is None and start_response is None:
            # Called as part of gunicorn's post_worker_init
            return _APP

    return _APP(environ, start_response)
예제 #26
0
def _statsd_host():
    from ichnaea.config import read_config
    conf = read_config()
    return conf.get_map('ichnaea').get('statsd_host', None)
예제 #27
0
def init_worker_process(signal, sender, **kw):  # pragma: no cover
    # called automatically when `celery worker` is started
    # get the app in the current worker process
    celery_app = app_or_default()
    conf = read_config()
    init_worker(celery_app, conf)
예제 #28
0
파일: base.py 프로젝트: amolk4games/ichnaea
    from unittest import TestCase

TEST_DIRECTORY = os.path.dirname(__file__)
DATA_DIRECTORY = os.path.join(TEST_DIRECTORY, 'data')
GEOIP_TEST_FILE = os.path.join(DATA_DIRECTORY, 'GeoIP2-City-Test.mmdb')
GEOIP_BAD_FILE = os.path.join(
    DATA_DIRECTORY, 'GeoIP2-Connection-Type-Test.mmdb')

SQLURI = os.environ.get('SQLURI')
REDIS_URI = os.environ.get('REDIS_URI')

SESSION = {}

# Some test-data constants

TEST_CONFIG = read_config(filename=os.path.join(DATA_DIRECTORY, 'test.ini'))

GEOIP_DATA = {
    'London': {
        'city': True,
        'region_code': 'GB',
        'region_name': 'United Kingdom',
        'ip': '81.2.69.192',
        'latitude': 51.5142,
        'longitude': -0.0931,
        'radius': CITY_RADII[2643743],
        'region_radius': GEOCODER.region_max_radius('GB'),
        'score': 0.8,
    },
    'Bhutan': {
        'city': False,
예제 #29
0
def _statsd_host():
    from ichnaea.config import read_config
    conf = read_config()
    return conf.get_map('ichnaea').get('statsd_host', None)
예제 #30
0
def main(argv, _db_rw=None, _raven_client=None, _stats_client=None):
    # run for example via:
    # bin/location_map --create --upload --datamaps=/path/to/datamaps/ \
    #   --output=ichnaea/content/static/tiles/

    parser = argparse.ArgumentParser(
        prog=argv[0], description='Generate and upload datamap tiles.')

    parser.add_argument('--create', action='store_true', help='Create tiles.')
    parser.add_argument('--upload',
                        action='store_true',
                        help='Upload tiles to S3.')
    parser.add_argument('--concurrency',
                        default=2,
                        help='How many concurrent render processes to use?')
    parser.add_argument('--datamaps', help='Directory of the datamaps tools.')
    parser.add_argument('--output',
                        help='Optional directory for local tile output.')

    args = parser.parse_args(argv[1:])

    if args.create:
        conf = read_config()
        if _db_rw:
            db = _db_rw
        else:  # pragma: no cover
            db = Database(conf.get('ichnaea', 'db_master'))
        bucketname = conf.get('ichnaea', 's3_assets_bucket').strip('/')
        raven_client = configure_raven(conf.get('ichnaea', 'sentry_dsn'),
                                       _client=_raven_client)
        stats_client = configure_stats(conf.get('ichnaea', 'statsd_host'),
                                       _client=_stats_client)

        upload = False
        if args.upload:  # pragma: no cover
            upload = bool(args.upload)

        concurrency = 2
        if args.concurrency:
            concurrency = int(args.concurrency)

        datamaps = ''
        if args.datamaps:
            datamaps = os.path.abspath(args.datamaps)

        output = None
        if args.output:
            output = os.path.abspath(args.output)

        try:
            with stats_client.timer("datamaps.total_time"):
                generate(db,
                         bucketname,
                         raven_client,
                         stats_client,
                         upload=upload,
                         concurrency=concurrency,
                         datamaps=datamaps,
                         output=output)
        except Exception:  # pragma: no cover
            raven_client.captureException()
            raise
    else:  # pragma: no cover
        parser.print_help()
예제 #31
0
파일: app.py 프로젝트: awoland/ichnaea
def init_worker_process(signal, sender, **kw):  # pragma: no cover
    # called automatically when `celery worker` is started
    # get the app in the current worker process
    celery_app = app_or_default()
    conf = read_config()
    init_worker(celery_app, conf)
예제 #32
0
파일: base.py 프로젝트: amolk4games/ichnaea
    from unittest import TestCase

TEST_DIRECTORY = os.path.dirname(__file__)
DATA_DIRECTORY = os.path.join(TEST_DIRECTORY, 'data')
GEOIP_TEST_FILE = os.path.join(DATA_DIRECTORY, 'GeoIP2-City-Test.mmdb')
GEOIP_BAD_FILE = os.path.join(DATA_DIRECTORY,
                              'GeoIP2-Connection-Type-Test.mmdb')

SQLURI = os.environ.get('SQLURI')
REDIS_URI = os.environ.get('REDIS_URI')

SESSION = {}

# Some test-data constants

TEST_CONFIG = read_config(filename=os.path.join(DATA_DIRECTORY, 'test.ini'))

GEOIP_DATA = {
    'London': {
        'city': True,
        'region_code': 'GB',
        'region_name': 'United Kingdom',
        'ip': '81.2.69.192',
        'latitude': 51.5142,
        'longitude': -0.0931,
        'radius': CITY_RADII[2643743],
        'region_radius': GEOCODER.region_max_radius('GB'),
        'score': 0.8,
    },
    'Bhutan': {
        'city': False,