Ejemplo n.º 1
0
def main(argv, _db_master=None, _heka_client=None, _stats_client=None):
    parser = argparse.ArgumentParser(
        prog=argv[0], description='Location Importer')

    parser.add_argument('source', help="The source file.")
    parser.add_argument('--userid', default=None,
                        help='Internal userid for attribution.')

    args = parser.parse_args(argv[1:])
    userid = None
    if args.userid is not None:
        userid = int(args.userid)

    conf = read_config()
    settings = conf.get_map('ichnaea')
    configure_heka(conf.filename, _heka_client=_heka_client)
    configure_stats(conf.get('ichnaea', 'statsd_host'), _client=_stats_client)

    # configure databases incl. test override hooks
    if _db_master is None:  # pragma: no cover
        db = Database(settings['db_master'])
    else:
        db = _db_master
    session = db.session()
    added = load_file(session, args.source, userid=userid)
    print('Added a total of %s records.' % added)
    session.commit()
    return added
Ejemplo n.º 2
0
def main(argv, _db_master=None):
    parser = argparse.ArgumentParser(
        prog=argv[0], description='Location Importer')

    parser.add_argument('source', help="The source file.")
    parser.add_argument('--userid', default=None,
                        help='Internal userid for attribution.')

    args = parser.parse_args(argv[1:])
    userid = None
    if args.userid is not None:
        userid = int(args.userid)

    settings = config().get_map('ichnaea')

    # configure databases incl. test override hooks
    if _db_master is None:
        db = Database(
            settings['db_master'],
            socket=settings.get('db_master_socket'),
            create=False,
        )
    else:
        db = _db_master
    session = db.session()
    added = load_file(session, args.source, userid=userid)
    print('Added a total of %s records.' % added)
    session.commit()
    return added
Ejemplo n.º 3
0
def main(argv, _db_master=None, _heka_client=None, _stats_client=None):
    parser = argparse.ArgumentParser(prog=argv[0],
                                     description='Location Importer')

    parser.add_argument('source', help="The source file.")
    parser.add_argument('--userid',
                        default=None,
                        help='Internal userid for attribution.')

    args = parser.parse_args(argv[1:])
    userid = None
    if args.userid is not None:
        userid = int(args.userid)

    conf = read_config()
    settings = conf.get_map('ichnaea')
    configure_heka(conf.filename, _heka_client=_heka_client)
    configure_stats(conf.get('ichnaea', 'statsd_host'), _client=_stats_client)

    # configure databases incl. test override hooks
    if _db_master is None:  # pragma: no cover
        db = Database(settings['db_master'])
    else:
        db = _db_master
    session = db.session()
    added = load_file(session, args.source, userid=userid)
    print('Added a total of %s records.' % added)
    session.commit()
    return added
Ejemplo n.º 4
0
def main(argv, _db_master=None):
    parser = argparse.ArgumentParser(prog=argv[0],
                                     description='Location Importer')

    parser.add_argument('source', help="The source file.")
    parser.add_argument('--userid',
                        default=None,
                        help='Internal userid for attribution.')

    args = parser.parse_args(argv[1:])
    userid = None
    if args.userid is not None:
        userid = int(args.userid)

    settings = config().get_map('ichnaea')

    # configure databases incl. test override hooks
    if _db_master is None:
        db = Database(
            settings['db_master'],
            socket=settings.get('db_master_socket'),
            create=False,
        )
    else:
        db = _db_master
    session = db.session()
    added = load_file(session, args.source, userid=userid)
    print('Added a total of %s records.' % added)
    session.commit()
    return added
Ejemplo n.º 5
0
    def setUp(self):
        TestCase.setUp(self)

        self.redis = VaurienRedis()
        self.mysql = VaurienMySQL()

        uri = os.environ.get('SQLURI',
                             'mysql+pymysql://root:mysql@localhost/location')
        self.db = Database(uri)

        self.install_apikey()
        self.install_fixtures()
Ejemplo n.º 6
0
def attach_database(app, settings=None, _db_master=None):
    # called manually during tests
    if _db_master is None:  # pragma: no cover
        db_master = Database(settings['db_master'])
    else:
        db_master = _db_master
    app.db_master = db_master
Ejemplo n.º 7
0
def main():

    settings = read_config().get_map('ichnaea')
    db = Database(settings['db_slave'])
    session = db.session()

    bad = []

    offset = 0
    count = 10000
    results = True
    while results:
        results = False
        r = session.execute("select id, lat, lon, mcc, mnc, lac, cid, radio, "
                            "total_measures from cell where "
                            "lat is not null and lon is not null and "
                            "mcc not in (1, 260) "
                            "order by id limit %d offset %d" %
                            (count, offset))
        offset += count
        for row in r:
            results = True
            (id, lat, lon, mcc, mnc, lac, cid, radio, total_measures) = row
            ccs = [c.alpha2 for c in mobile_codes.mcc(str(mcc))]
            if not any([location_is_in_country(lat, lon, c, 1) for c in ccs]):
                if ccs:
                    s = ",".join(ccs)
                else:
                    continue
                bad.append(dict(
                    type='Feature',
                    properties=dict(
                        mcc=mcc,
                        mnc=mnc,
                        lac=lac,
                        cid=cid,
                        radio=radio,
                        total_measures=total_measures,
                        countries=s),
                    geometry=dict(
                        type='Point',
                        coordinates=[lon, lat])))

    json.dump(dict(type='FeatureCollection',
                   features=bad),
              sys.stdout,
              indent=True)
Ejemplo n.º 8
0
def attach_database(app, _db_master=None):
    # called manually during tests
    settings = config().get_map('ichnaea')
    if _db_master is None:  # pragma: no cover
        db_master = Database(settings['db_master'])
    else:
        db_master = _db_master
    app.db_master = db_master
Ejemplo n.º 9
0
def main(global_config, _db_master=None, _db_slave=None, **settings):
    config = Configurator(settings=settings)
    config.include("cornice")
    settings = config.registry.settings

    # logging
    global logger
    logger.setLevel(logging.DEBUG)
    sh = logging.StreamHandler()
    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    sh.setFormatter(formatter)
    logger.addHandler(sh)
    waitress_log = logging.getLogger('waitress')
    waitress_log.addHandler(sh)

    from ichnaea.content.views import configure_content
    from ichnaea.service import configure_service

    configure_content(config)
    configure_service(config)

    # configure databases incl. test override hooks
    if _db_master is None:
        config.registry.db_master = Database(
            settings['db_master'],
            socket=settings.get('db_master_socket'),
        )
    else:
        config.registry.db_master = _db_master
    if _db_slave is None:
        config.registry.db_slave = Database(
            settings['db_slave'],
            socket=settings.get('db_slave_socket'),
            create=False,
        )
    else:
        config.registry.db_slave = _db_slave
    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_request_method(db_master_session, property=True)
    config.add_request_method(db_slave_session, property=True)

    # replace json renderer with decimal json variant
    config.add_renderer('json', decimaljson.Renderer())
    return config.make_wsgi_app()
Ejemplo n.º 10
0
def main(global_config, _db_master=None, _db_slave=None, **settings):
    config = Configurator(settings=settings)

    # add support for pt templates
    config.include('pyramid_chameleon')

    settings = config.registry.settings

    from ichnaea.content.views import configure_content
    from ichnaea.service import configure_service
    from ichnaea.heka_logging import configure_heka

    configure_content(config)
    configure_service(config)

    # configure databases incl. test override hooks
    if _db_master is None:
        config.registry.db_master = Database(
            settings['db_master'],
            socket=settings.get('db_master_socket'),
        )
    else:
        config.registry.db_master = _db_master
    if _db_slave is None:
        config.registry.db_slave = Database(
            settings['db_slave'],
            socket=settings.get('db_slave_socket'),
            create=False,
        )
    else:
        config.registry.db_slave = _db_slave

    config.registry.geoip_db = configure_geoip(config.registry.settings)

    config.registry.heka_client = configure_heka(config.registry.settings)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.heka_logging.heka_tween_factory', under=EXCVIEW)
    config.add_request_method(db_master_session, property=True)
    config.add_request_method(db_slave_session, property=True)

    # replace json renderer with decimal json variant
    config.add_renderer('json', decimaljson.Renderer())
    return config.make_wsgi_app()
Ejemplo n.º 11
0
def main(argv):
    parser = argparse.ArgumentParser(prog=argv[0], description="Location Importer")

    parser.add_argument("--dry-run", action="store_true")
    # TODO rely on ICHNAEA_CFG / ichnaea.config, as the worker is relying
    # on it anyways
    parser.add_argument("config", help="config file")
    parser.add_argument("source", help="source file")
    args = parser.parse_args(argv[1:])
    settings = Config(args.config).get_map("ichnaea")
    db = Database(settings["db_master"], socket=settings.get("db_master_socket"), create=False)
    session = db.session()
    added = load_file(session, args.source)
    print("Added %s records." % added)
    if args.dry_run:
        session.rollback()
    else:  # pragma: no cover
        session.commit()
    return added
Ejemplo n.º 12
0
def main():

    settings = read_config().get_map('ichnaea')
    db = Database(settings['db_slave'])
    session = db.session()

    bad = []

    offset = 0
    count = 10000
    results = True
    while results:
        results = False
        r = session.execute("select id, lat, lon, mcc, mnc, lac, cid, radio, "
                            "total_measures from cell where "
                            "lat is not null and lon is not null and "
                            "mcc not in (1, 260) "
                            "order by id limit %d offset %d" % (count, offset))
        offset += count
        for row in r:
            results = True
            (id, lat, lon, mcc, mnc, lac, cid, radio, total_measures) = row
            ccs = [c.alpha2 for c in mobile_codes.mcc(str(mcc))]
            if not any([location_is_in_country(lat, lon, c, 1) for c in ccs]):
                if ccs:
                    s = ",".join(ccs)
                else:
                    continue
                bad.append(
                    dict(type='Feature',
                         properties=dict(mcc=mcc,
                                         mnc=mnc,
                                         lac=lac,
                                         cid=cid,
                                         radio=radio,
                                         total_measures=total_measures,
                                         countries=s),
                         geometry=dict(type='Point', coordinates=[lon, lat])))

    json.dump(dict(type='FeatureCollection', features=bad),
              sys.stdout,
              indent=True)
Ejemplo n.º 13
0
    def setUp(self):
        TestCase.setUp(self)

        self.redis = VaurienRedis()
        self.mysql = VaurienMySQL()

        uri = os.environ.get('SQLURI',
                             'mysql+pymysql://root:mysql@localhost/location')
        self.db = Database(uri)

        self.install_apikey()
        self.install_fixtures()
Ejemplo n.º 14
0
def main(argv, _db_master=None):
    parser = argparse.ArgumentParser(
        prog=argv[0], description='Location Importer')

    parser.add_argument('source', help="source file")
    args = parser.parse_args(argv[1:])
    settings = config().get_map('ichnaea')

    # configure databases incl. test override hooks
    if _db_master is None:
        db = Database(
            settings['db_master'],
            socket=settings.get('db_master_socket'),
            create=False,
        )
    else:
        db = _db_master
    session = db.session()
    added = load_file(session, args.source)
    print('Added %s records.' % added)
    session.commit()
    return added
Ejemplo n.º 15
0
def main(argv, _db_master=None):
    parser = argparse.ArgumentParser(prog=argv[0],
                                     description='Location Importer')

    parser.add_argument('source', help="source file")
    args = parser.parse_args(argv[1:])
    settings = config().get_map('ichnaea')

    # configure databases incl. test override hooks
    if _db_master is None:
        db = Database(
            settings['db_master'],
            socket=settings.get('db_master_socket'),
            create=False,
        )
    else:
        db = _db_master
    session = db.session()
    added = load_file(session, args.source)
    print('Added %s records.' % added)
    session.commit()
    return added
Ejemplo n.º 16
0
class DBFixture(object):

    def __init__(self):
        self.db_master = Database(SQLURI)
        master_conn = self.db_master.engine.connect()
        self.master_trans = master_conn.begin()
        self.db_master_session = self.db_master.session()

    def install_wifi_aps(self):
        self.db_master_session.execute(Wifi.__table__.delete())

        ap_data = json.load(open(AP_FILE),
                            object_hook=JSONLocationDictDecoder)

        session = self.db_master_session
        data = []
        for i, ((lat, lon), ap_set) in enumerate(ap_data.items()):
            for ap in ap_set:
                wifi = Wifi(key=ap['key'].replace(":", ""),
                            lat=lat * (10 ** 7),
                            lon=lon * (10 ** 7))
                data.append(wifi)
            session.add_all(data)
            session.flush()
            data = []
        session.commit()

    def install_cell_towers(self):
        self.db_master_session.execute(Cell.__table__.delete())

        tower_data = json.load(open(TOWER_FILE),
                               object_hook=JSONLocationDictDecoder)

        session = self.db_master_session

        data = []
        for i, ((lat, lon), cell_data_set) in enumerate(tower_data.items()):
            for cell_data in cell_data_set:
                data.append(Cell(lat=lat * (10 ** 7),
                                 lon=lon * (10 ** 7),
                                 radio=cell_data['radio'],
                                 cid=cell_data['cid'],
                                 mcc=cell_data['mcc'],
                                 mnc=cell_data['mnc'],
                                 lac=cell_data['lac']))
            session.add_all(data)
            session.flush()
            data = []
        session.commit()
Ejemplo n.º 17
0
def main(argv, _db_rw=None, _raven_client=None):
    parser = argparse.ArgumentParser(prog=argv[0],
                                     description='Initialize Ichnaea database')

    parser.add_argument('--alembic_ini',
                        help='Path to the alembic migration config.')
    parser.add_argument('--location_ini',
                        help='Path to the ichnaea app config.')
    parser.add_argument('--initdb',
                        action='store_true',
                        help='Initialize database')

    args = parser.parse_args(argv[1:])

    if args.initdb:
        # Either use explicit config file location or fallback
        # on environment variable or finally file in current directory
        if not args.location_ini:
            location_ini = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini')
        else:
            location_ini = args.location_ini
        location_ini = os.path.abspath(location_ini)
        location_cfg = read_config(filename=location_ini)

        # Either use explicit config file location or fallback
        # to a file in the same directory as the ichnaea.ini
        if not args.alembic_ini:
            alembic_ini = os.path.join(os.path.dirname(location_ini),
                                       'alembic.ini')
        else:
            alembic_ini = args.alembic_ini
        alembic_ini = os.path.abspath(alembic_ini)
        alembic_cfg = Config(alembic_ini)
        alembic_section = alembic_cfg.get_section('alembic')

        if _db_rw is None:
            db_rw = Database(alembic_section['sqlalchemy.url'])
        else:
            db_rw = _db_rw
        configure_raven(location_cfg.get('ichnaea', 'sentry_dsn'),
                        _client=_raven_client)

        engine = db_rw.engine
        create_schema(engine, alembic_cfg, location_cfg)
    else:
        parser.print_help()
Ejemplo n.º 18
0
class TestSearch(TestCase):
    """
    Search tests should only be affected by mysql outages.

    All redis tests are in one test case
    """

    def setUp(self):
        TestCase.setUp(self)

        self.redis = VaurienRedis()
        self.mysql = VaurienMySQL()

        uri = os.environ.get('SQLURI',
                             'mysql+pymysql://root:mysql@localhost/location')
        self.db = Database(uri)

        self.install_apikey()
        self.install_fixtures()

    def install_fixtures(self):
        session = self.db.session()
        PARIS_LAT_DEG = from_degrees(PARIS_LAT)
        PARIS_LON_DEG = from_degrees(PARIS_LON)
        qry = session.query(Cell)
        if qry.count() > 0:
            session.query(Cell).delete()

        lat = from_degrees(PARIS_LAT)
        lon = from_degrees(PARIS_LON)

        key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
        data = [
            Cell(lat=lat, lon=lon, radio=2, cid=4, **key),
            Cell(lat=lat + 20000, lon=lon + 40000, radio=2, cid=5, **key),
        ]
        session.add_all(data)

        if session.query(Wifi).count() > 0:
            session.query(Wifi).delete()

        wifis = [
            Wifi(key="A1", lat=PARIS_LAT_DEG, lon=PARIS_LON_DEG),
            Wifi(key="B2", lat=PARIS_LAT_DEG, lon=PARIS_LON_DEG),
            Wifi(key="C3", lat=PARIS_LAT_DEG, lon=PARIS_LON_DEG),
            Wifi(key="D4", lat=None, lon=None),
        ]
        session.add_all(wifis)
        session.commit()

    def install_apikey(self):
        session = self.db.session()
        if session.query(ApiKey).filter(
           ApiKey.valid_key == 'test').count() > 0:
            session.query(ApiKey).delete()

        session.add(ApiKey(valid_key='test', maxreq=0))
        session.commit()

    def test_mysql_dummy(self):
        # this should pass, otherwise, vaurien has screwed up
        self.mysql.dummy()
        self.redis.dummy()
        status_code, content = do_search(use_ip=PARIS_IP)
        eq_(status_code, 200)
        expected = {u'status': u'ok',
                    u'lat': PARIS_LAT,
                    u'lon': PARIS_LON,
                    u'accuracy': 100}
        actual = json.loads(content)

        self.assertAlmostEquals(actual['status'], expected['status'])
        self.assertAlmostEquals(actual['lat'], expected['lat'])
        self.assertAlmostEquals(actual['lon'], expected['lon'])
        self.assertAlmostEquals(actual['accuracy'], expected['accuracy'])

    def test_mysql_delay(self):
        # this should pass, otherwise, vaurien has screwed up
        self.mysql.delay()
        self.redis.dummy()
        start = time.time()
        status_code, content = do_search()
        end = time.time()
        assert (end-start) > 1.0
        eq_(status_code, 200)
        eq_(json.loads(content), {"status": "ok",
                                  "lat": PARIS_LAT,
                                  "lon": PARIS_LON,
                                  "accuracy": 100})

    def test_mysql_blackout(self):
        # This test has been renamed so that it runs last
        self.mysql.blackout()
        self.redis.dummy()

        # MySQL blackouts will cause API key checking to be disabled
        status_code, content = do_search(apikey='invalid_key',
                                         use_ip=FREMONT_IP)

        # MySQL blackouts will force only geo-ip to work
        eq_(status_code, 200)
        actual = json.loads(content)
        expected = {"status": "ok",
                    "lat": FREMONT_LAT,
                    "lon": FREMONT_LON,
                    "accuracy": GEOIP_CITY_ACCURACY}

        # TODO: not sure why we need almost equal for geoip
        self.assertAlmostEquals(actual['status'], expected['status'])
        self.assertAlmostEquals(actual['lat'], expected['lat'])
        self.assertAlmostEquals(actual['lon'], expected['lon'])
        self.assertAlmostEquals(actual['accuracy'], expected['accuracy'])

    def test_redis_dummy(self):
        self.mysql.dummy()
        self.redis.dummy()
        status_code, content = do_search()
        eq_(status_code, 200)
        eq_(json.loads(content), {"status": "ok",
                                  "lat": PARIS_LAT,
                                  "lon": PARIS_LON,
                                  "accuracy": 100})

    def test_redis_delay(self):
        self.mysql.dummy()
        self.redis.delay()
        start = time.time()
        status_code, content = do_search()
        delta = time.time() - start
        # The delay in redis should not affect search
        self.assertTrue(delta < 1.0)
        eq_(status_code, 200)
        eq_(json.loads(content), {"status": "ok",
                                  "lat": PARIS_LAT,
                                  "lon": PARIS_LON,
                                  "accuracy": 100})

    def test_redis_blackout(self):
        self.mysql.dummy()
        self.redis.blackout()

        start = time.time()
        status_code, content = do_search()
        delta = time.time() - start
        # The redis blackout should not affect search at all
        self.assertTrue(delta < 1.0)
        eq_(status_code, 200)
        eq_(json.loads(content), {"status": "ok",
                                  "lat": PARIS_LAT,
                                  "lon": PARIS_LON,
                                  "accuracy": 100})
Ejemplo n.º 19
0
def _make_db(uri=SQLURI):
    return Database(uri)
Ejemplo n.º 20
0
class TestSearch(TestCase):
    """
    Search tests should only be affected by mysql outages.

    All redis tests are in one test case
    """
    def setUp(self):
        TestCase.setUp(self)

        self.redis = VaurienRedis()
        self.mysql = VaurienMySQL()

        uri = os.environ.get('SQLURI',
                             'mysql+pymysql://root:mysql@localhost/location')
        self.db = Database(uri)

        self.install_apikey()
        self.install_fixtures()

    def install_fixtures(self):
        session = self.db.session()
        PARIS_LAT_DEG = from_degrees(PARIS_LAT)
        PARIS_LON_DEG = from_degrees(PARIS_LON)
        qry = session.query(Cell)
        if qry.count() > 0:
            session.query(Cell).delete()

        lat = from_degrees(PARIS_LAT)
        lon = from_degrees(PARIS_LON)

        key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
        data = [
            Cell(lat=lat, lon=lon, radio=2, cid=4, **key),
            Cell(lat=lat + 20000, lon=lon + 40000, radio=2, cid=5, **key),
        ]
        session.add_all(data)

        if session.query(Wifi).count() > 0:
            session.query(Wifi).delete()

        wifis = [
            Wifi(key="A1", lat=PARIS_LAT_DEG, lon=PARIS_LON_DEG),
            Wifi(key="B2", lat=PARIS_LAT_DEG, lon=PARIS_LON_DEG),
            Wifi(key="C3", lat=PARIS_LAT_DEG, lon=PARIS_LON_DEG),
            Wifi(key="D4", lat=None, lon=None),
        ]
        session.add_all(wifis)
        session.commit()

    def install_apikey(self):
        session = self.db.session()
        if session.query(ApiKey).filter(
                ApiKey.valid_key == 'test').count() > 0:
            session.query(ApiKey).delete()

        session.add(ApiKey(valid_key='test', maxreq=0))
        session.commit()

    def test_mysql_dummy(self):
        # this should pass, otherwise, vaurien has screwed up
        self.mysql.dummy()
        self.redis.dummy()
        status_code, content = do_search(use_ip=PARIS_IP)
        eq_(status_code, 200)
        expected = {
            u'status': u'ok',
            u'lat': PARIS_LAT,
            u'lon': PARIS_LON,
            u'accuracy': 100
        }
        actual = json.loads(content)

        self.assertAlmostEquals(actual['status'], expected['status'])
        self.assertAlmostEquals(actual['lat'], expected['lat'])
        self.assertAlmostEquals(actual['lon'], expected['lon'])
        self.assertAlmostEquals(actual['accuracy'], expected['accuracy'])

    def test_mysql_delay(self):
        # this should pass, otherwise, vaurien has screwed up
        self.mysql.delay()
        self.redis.dummy()
        start = time.time()
        status_code, content = do_search()
        end = time.time()
        assert (end - start) > 1.0
        eq_(status_code, 200)
        eq_(json.loads(content), {
            "status": "ok",
            "lat": PARIS_LAT,
            "lon": PARIS_LON,
            "accuracy": 100
        })

    def test_mysql_blackout(self):
        # This test has been renamed so that it runs last
        self.mysql.blackout()
        self.redis.dummy()

        # MySQL blackouts will cause API key checking to be disabled
        status_code, content = do_search(apikey='invalid_key',
                                         use_ip=FREMONT_IP)

        # MySQL blackouts will force only geo-ip to work
        eq_(status_code, 200)
        actual = json.loads(content)
        expected = {
            "status": "ok",
            "lat": FREMONT_LAT,
            "lon": FREMONT_LON,
            "accuracy": GEOIP_CITY_ACCURACY
        }

        # TODO: not sure why we need almost equal for geoip
        self.assertAlmostEquals(actual['status'], expected['status'])
        self.assertAlmostEquals(actual['lat'], expected['lat'])
        self.assertAlmostEquals(actual['lon'], expected['lon'])
        self.assertAlmostEquals(actual['accuracy'], expected['accuracy'])

    def test_redis_dummy(self):
        self.mysql.dummy()
        self.redis.dummy()
        status_code, content = do_search()
        eq_(status_code, 200)
        eq_(json.loads(content), {
            "status": "ok",
            "lat": PARIS_LAT,
            "lon": PARIS_LON,
            "accuracy": 100
        })

    def test_redis_delay(self):
        self.mysql.dummy()
        self.redis.delay()
        start = time.time()
        status_code, content = do_search()
        delta = time.time() - start
        # The delay in redis should not affect search
        self.assertTrue(delta < 1.0)
        eq_(status_code, 200)
        eq_(json.loads(content), {
            "status": "ok",
            "lat": PARIS_LAT,
            "lon": PARIS_LON,
            "accuracy": 100
        })

    def test_redis_blackout(self):
        self.mysql.dummy()
        self.redis.blackout()

        start = time.time()
        status_code, content = do_search()
        delta = time.time() - start
        # The redis blackout should not affect search at all
        self.assertTrue(delta < 1.0)
        eq_(status_code, 200)
        eq_(json.loads(content), {
            "status": "ok",
            "lat": PARIS_LAT,
            "lon": PARIS_LON,
            "accuracy": 100
        })
Ejemplo n.º 21
0
 def __init__(self):
     self.db_master = Database(SQLURI)
     master_conn = self.db_master.engine.connect()
     self.master_trans = master_conn.begin()
     self.db_master_session = self.db_master.session()
Ejemplo n.º 22
0
def main(argv, _db_master=None,
         _heka_client=None, _stats_client=None):
    # run for example via:
    # bin/location_map --create --upload --datamaps=/path/to/datamaps/ \
    #   --output=ichnaea/content/static/tiles/

    parser = argparse.ArgumentParser(
        prog=argv[0], description='Generate and upload datamap tiles.')

    parser.add_argument('--create', action='store_true',
                        help='Create tiles.')
    parser.add_argument('--upload', action='store_true',
                        help='Upload tiles to S3.')
    parser.add_argument('--concurrency', default=2,
                        help='How many concurrent render processes to use?')
    parser.add_argument('--datamaps',
                        help='Directory of the datamaps tools.')
    parser.add_argument('--output',
                        help='Optional directory for local tile output.')

    args = parser.parse_args(argv[1:])

    if args.create:
        conf = read_config()
        if _db_master:
            db = _db_master
        else:  # pragma: no cover
            db = Database(conf.get('ichnaea', 'db_master'))
        bucketname = conf.get('ichnaea', 's3_assets_bucket').strip('/')
        heka_client = configure_heka(conf.filename, _heka_client=_heka_client)
        stats_client = configure_stats(
            conf.get('ichnaea', 'statsd_host'), _client=_stats_client)

        upload = False
        if args.upload:  # pragma: no cover
            upload = bool(args.upload)

        concurrency = 2
        if args.concurrency:
            concurrency = int(args.concurrency)

        datamaps = ''
        if args.datamaps:
            datamaps = os.path.abspath(args.datamaps)

        output = None
        if args.output:
            output = os.path.abspath(args.output)

        try:
            with stats_client.timer("datamaps.total_time"):
                generate(db, bucketname, heka_client, stats_client,
                         upload=upload,
                         concurrency=concurrency,
                         datamaps=datamaps,
                         output=output)
        except Exception:  # pragma: no cover
            heka_client.raven(RAVEN_ERROR)
            raise
    else:  # pragma: no cover
        parser.print_help()
Ejemplo n.º 23
0
def _make_db(create=True):
    return Database(SQLURI, socket=SQLSOCKET, create=create)
Ejemplo n.º 24
0
def main(global_config,
         heka_config=None,
         init=False,
         _db_master=None,
         _db_slave=None,
         _heka_client=None,
         _redis=None,
         _stats_client=None,
         **settings):
    config = Configurator(settings=settings)

    # add support for pt templates
    config.include('pyramid_chameleon')

    settings = config.registry.settings

    from ichnaea.content.views import configure_content
    from ichnaea.logging import configure_heka
    from ichnaea.logging import configure_stats
    from ichnaea.service import configure_service

    configure_content(config)
    configure_service(config)

    # configure databases incl. test override hooks
    if _db_master is None:
        config.registry.db_master = Database(settings['db_master'])
    else:
        config.registry.db_master = _db_master
    if _db_slave is None:
        config.registry.db_slave = Database(settings['db_slave'])
    else:
        config.registry.db_slave = _db_slave

    if _redis is None:
        config.registry.redis_client = None
        if 'redis_url' in settings:
            config.registry.redis_client = redis_client(settings['redis_url'])
    else:
        config.registry.redis_client = _redis

    if _heka_client is None:  # pragma: no cover
        config.registry.heka_client = heka_client = configure_heka(heka_config)
    else:
        config.registry.heka_client = heka_client = _heka_client

    config.registry.stats_client = configure_stats(settings.get('statsd_host'),
                                                   _client=_stats_client)

    config.registry.geoip_db = configure_geoip(config.registry.settings,
                                               heka_client=heka_client)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.logging.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_master_session, property=True)
    config.add_request_method(db_slave_session, property=True)

    # replace json renderer with custom json variant
    config.add_renderer('json', customjson.Renderer())

    # Should we try to initialize and establish the outbound connections?
    if init:  # pragma: no cover
        registry = config.registry
        registry.db_slave.ping()
        registry.redis_client.ping()
        registry.stats_client.ping()

    return config.make_wsgi_app()
Ejemplo n.º 25
0
def main(global_config,
         heka_config=None,
         init=False,
         _db_master=None,
         _db_slave=None,
         _heka_client=None,
         _redis=None,
         _stats_client=None,
         **settings):
    config = Configurator(settings=settings)

    # add support for pt templates
    config.include('pyramid_chameleon')

    settings = config.registry.settings

    from ichnaea.content.views import configure_content
    from ichnaea.logging import configure_heka
    from ichnaea.logging import configure_stats
    from ichnaea.service import configure_service

    configure_content(config)
    configure_service(config)

    # configure databases incl. test override hooks
    if _db_master is None:
        config.registry.db_master = Database(settings['db_master'])
    else:
        config.registry.db_master = _db_master
    if _db_slave is None:
        config.registry.db_slave = Database(settings['db_slave'])
    else:
        config.registry.db_slave = _db_slave

    if _redis is None:
        config.registry.redis_client = None
        if 'redis_url' in settings:
            config.registry.redis_client = redis_client(settings['redis_url'])
    else:
        config.registry.redis_client = _redis

    if _heka_client is None:  # pragma: no cover
        config.registry.heka_client = heka_client = configure_heka(heka_config)
    else:
        config.registry.heka_client = heka_client = _heka_client

    config.registry.stats_client = configure_stats(settings.get('statsd_host'),
                                                   _client=_stats_client)

    config.registry.geoip_db = configure_geoip(config.registry.settings,
                                               heka_client=heka_client)

    config.add_tween('ichnaea.db.db_tween_factory', under=EXCVIEW)
    config.add_tween('ichnaea.logging.log_tween_factory', under=EXCVIEW)
    config.add_request_method(db_master_session, property=True)
    config.add_request_method(db_slave_session, property=True)

    # replace json renderer with custom json variant
    config.add_renderer('json', customjson.Renderer())

    # Should we try to initialize and establish the outbound connections?
    if init:  # pragma: no cover
        # Test the slave DB connection
        with db_worker_session(config.registry.db_slave) as session:
            try:
                session.execute(select([func.now()])).first()
            except OperationalError:
                # Let the instance start, so it can recover / reconnect
                # to the DB later, but provide degraded service in the
                # meantime.
                pass

        # Test the redis connection
        try:
            config.registry.redis_client.ping()
        except ConnectionError:
            # Same as for the DB, continue with degraded service.
            pass

    return config.make_wsgi_app()
Ejemplo n.º 26
0
def _make_db(uri=SQLURI, socket=SQLSOCKET, create=True):
    return Database(uri, socket=socket, create=create)