def main(argv, _db_master=None, _heka_client=None, _stats_client=None):
parser = argparse.ArgumentParser(
prog=argv[0], description='Location Importer')
parser.add_argument('source', help="The source file.")
parser.add_argument('--userid', default=None,
help='Internal userid for attribution.')
args = parser.parse_args(argv[1:])
userid = None
if args.userid is not None:
userid = int(args.userid)
conf = read_config()
settings = conf.get_map('ichnaea')
configure_heka(conf.filename, _heka_client=_heka_client)
configure_stats(conf.get('ichnaea', 'statsd_host'), _client=_stats_client)
# configure databases incl. test override hooks
if _db_master is None: # pragma: no cover
db = Database(settings['db_master'])
else:
db = _db_master
session = db.session()
added = load_file(session, args.source, userid=userid)
print('Added a total of %s records.' % added)
session.commit()
return added
def main(argv, _db_master=None, _heka_client=None, _stats_client=None):
parser = argparse.ArgumentParser(prog=argv[0],
description='Location Importer')
parser.add_argument('source', help="The source file.")
parser.add_argument('--userid',
default=None,
help='Internal userid for attribution.')
args = parser.parse_args(argv[1:])
userid = None
if args.userid is not None:
userid = int(args.userid)
conf = read_config()
settings = conf.get_map('ichnaea')
configure_heka(conf.filename, _heka_client=_heka_client)
configure_stats(conf.get('ichnaea', 'statsd_host'), _client=_stats_client)
# configure databases incl. test override hooks
if _db_master is None: # pragma: no cover
db = Database(settings['db_master'])
else:
db = _db_master
session = db.session()
added = load_file(session, args.source, userid=userid)
print('Added a total of %s records.' % added)
session.commit()
return added
def main(argv, _db_master=None,
_heka_client=None, _stats_client=None): # pragma: no cover
# run for example via:
# bin/location_map --create --upload --datamaps=/path/to/datamaps/ \
# --output=ichnaea/content/static/tiles/
parser = argparse.ArgumentParser(
prog=argv[0], description='Generate and upload datamap tiles.')
parser.add_argument('--create', action='store_true',
help='Create tiles.')
parser.add_argument('--upload', action='store_true',
help='Upload tiles to S3.')
parser.add_argument('--concurrency', default=2,
help='How many concurrent render processes to use?')
parser.add_argument('--datamaps',
help='Directory of the datamaps tools.')
parser.add_argument('--output',
help='Optional directory for local tile output.')
args = parser.parse_args(argv[1:])
if args.create:
conf = read_config()
db = Database(conf.get('ichnaea', 'db_master'))
bucketname = conf.get('ichnaea', 's3_assets_bucket').strip('/')
heka_client = configure_heka(conf.filename, _heka_client=_heka_client)
stats_client = configure_stats(
conf.get('ichnaea', 'statsd_host'), _client=_stats_client)
upload = False
if args.upload:
upload = bool(args.upload)
concurrency = 2
if args.concurrency:
concurrency = int(args.concurrency)
datamaps = ''
if args.datamaps:
datamaps = os.path.abspath(args.datamaps)
output = None
if args.output:
output = os.path.abspath(args.output)
try:
with stats_client.timer("datamaps.total_time"):
generate(db, bucketname, heka_client, stats_client,
upload=upload,
concurrency=concurrency,
datamaps=datamaps,
output=output)
except Exception:
heka_client.raven(RAVEN_ERROR)
raise
else:
parser.print_help()
def configure_celery(celery):
conf = read_config()
if conf.has_section('celery'):
section = conf.get_map('celery')
else: # pragma: no cover
# happens while building docs locally and on rtfd.org
return
# testing settings
always_eager = bool(os.environ.get('CELERY_ALWAYS_EAGER', False))
redis_uri = os.environ.get('REDIS_URI', 'redis://localhost:6379/1')
if always_eager and redis_uri:
broker_url = redis_uri
result_url = redis_uri
else: # pragma: no cover
broker_url = section['broker_url']
result_url = section['result_url']
if 'redis' in broker_url:
broker_options = {}
# Based on celery / redis caveats
# celery.rtfd.org/en/latest/getting-started/brokers/redis.html#caveats
broker_options['fanout_patterns'] = True
broker_options['fanout_prefix'] = True
broker_options['visibility_timeout'] = 3600
if 'redis' in result_url:
celery.conf.update(
CELERY_RESULT_BACKEND=result_url,
)
celery.conf.update(
# testing
CELERY_ALWAYS_EAGER=always_eager,
CELERY_EAGER_PROPAGATES_EXCEPTIONS=always_eager,
# broker
BROKER_URL=broker_url,
BROKER_TRANSPORT_OPTIONS=broker_options,
# queues
CELERY_DEFAULT_QUEUE='default',
CELERY_QUEUES=CELERY_QUEUES,
# tasks
CELERY_IMPORTS=CELERY_IMPORTS,
# forward compatibility
CELERYD_FORCE_EXECV=True,
# optimization
CELERYD_PREFETCH_MULTIPLIER=8,
CELERY_DISABLE_RATE_LIMITS=True,
CELERY_MESSAGE_COMPRESSION='gzip',
# security
CELERY_ACCEPT_CONTENT=['json'],
CELERY_RESULT_SERIALIZER='json',
CELERY_TASK_SERIALIZER='json',
# schedule
CELERYBEAT_LOG_LEVEL="WARNING",
CELERYBEAT_SCHEDULE=CELERYBEAT_SCHEDULE,
)
def configure_celery(celery):
conf = read_config()
if conf.has_section('celery'):
section = conf.get_map('celery')
else: # pragma: no cover
# happens while building docs locally and on rtfd.org
return
# testing settings
always_eager = bool(os.environ.get('CELERY_ALWAYS_EAGER', False))
redis_uri = os.environ.get('REDIS_URI', 'redis://localhost:6379/1')
if always_eager and redis_uri:
broker_url = redis_uri
result_url = redis_uri
else: # pragma: no cover
broker_url = section['broker_url']
result_url = section['result_url']
if 'redis' in broker_url:
broker_options = {}
# Based on celery / redis caveats
# celery.rtfd.org/en/latest/getting-started/brokers/redis.html#caveats
broker_options['fanout_patterns'] = True
broker_options['fanout_prefix'] = True
broker_options['visibility_timeout'] = 3600
if 'redis' in result_url:
celery.conf.update(
CELERY_RESULT_BACKEND=result_url,
)
celery.conf.update(
# testing
CELERY_ALWAYS_EAGER=always_eager,
CELERY_EAGER_PROPAGATES_EXCEPTIONS=always_eager,
# broker
BROKER_URL=broker_url,
BROKER_TRANSPORT_OPTIONS=broker_options,
# queues
CELERY_DEFAULT_QUEUE='celery_default',
CELERY_QUEUES=CELERY_QUEUES,
# tasks
CELERY_IMPORTS=CELERY_IMPORTS,
# optimization
CELERYD_PREFETCH_MULTIPLIER=8,
CELERY_DISABLE_RATE_LIMITS=True,
CELERY_MESSAGE_COMPRESSION='gzip',
# security
CELERY_ACCEPT_CONTENT=['json'],
CELERY_RESULT_SERIALIZER='json',
CELERY_TASK_SERIALIZER='json',
# schedule
CELERYBEAT_LOG_LEVEL="WARNING",
CELERYBEAT_SCHEDULE=CELERYBEAT_SCHEDULE,
)
def init_worker_process(signal, sender, **kw): # pragma: no cover
# called automatically when `celery worker` is started
# get the app in the current worker process
app = app_or_default()
conf = read_config()
settings = conf.get_map('ichnaea')
attach_database(app, settings=settings)
attach_redis_client(app, settings=settings)
configure_s3_backup(app, settings=settings)
configure_ocid_import(app, settings=settings)
configure_heka(conf.filename)
configure_stats(settings.get('statsd_host'))
def application(environ, start_response): # pragma: no cover
# Actual WSGI application endpoint, used on the command line via:
# bin/gunicorn -c ichnaea.gunicorn_config ichnaea:application
global _APP
if _APP is None:
conf = read_config()
# Signal this call was made as part of app initialization
_APP = main({}, init=True, **conf.get_map('ichnaea'))
if environ is None and start_response is None:
# Called as part of gunicorn's post_worker_init
return _APP
return _APP(environ, start_response)
def application(environ, start_response): # pragma: no cover
# Actual WSGI application endpoint, used on the command line via:
# bin/gunicorn -c ichnaea.gunicorn_config ichnaea:application
global _APP
if _APP is None:
conf = read_config()
# Signal this call was made as part of app initialization
_APP = main({}, heka_config=conf.filename, init=True,
**conf.get_map('ichnaea'))
if environ is None and start_response is None:
# Called as part of gunicorn's post_worker_init
return _APP
return _APP(environ, start_response)
def main():
settings = read_config().get_map('ichnaea')
db = Database(settings['db_slave'])
session = db.session()
bad = []
offset = 0
count = 10000
results = True
while results:
results = False
r = session.execute("select id, lat, lon, mcc, mnc, lac, cid, radio, "
"total_measures from cell where "
"lat is not null and lon is not null and "
"mcc not in (1, 260) "
"order by id limit %d offset %d" %
(count, offset))
offset += count
for row in r:
results = True
(id, lat, lon, mcc, mnc, lac, cid, radio, total_measures) = row
ccs = [c.alpha2 for c in mobile_codes.mcc(str(mcc))]
if not any([location_is_in_country(lat, lon, c, 1) for c in ccs]):
if ccs:
s = ",".join(ccs)
else:
continue
bad.append(dict(
type='Feature',
properties=dict(
mcc=mcc,
mnc=mnc,
lac=lac,
cid=cid,
radio=radio,
total_measures=total_measures,
countries=s),
geometry=dict(
type='Point',
coordinates=[lon, lat])))
json.dump(dict(type='FeatureCollection',
features=bad),
sys.stdout,
indent=True)
def main(argv, _db_rw=None, _raven_client=None):
parser = argparse.ArgumentParser(prog=argv[0],
description='Initialize Ichnaea database')
parser.add_argument('--alembic_ini',
help='Path to the alembic migration config.')
parser.add_argument('--location_ini',
help='Path to the ichnaea app config.')
parser.add_argument('--initdb',
action='store_true',
help='Initialize database')
args = parser.parse_args(argv[1:])
if args.initdb:
# Either use explicit config file location or fallback
# on environment variable or finally file in current directory
if not args.location_ini:
location_ini = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini')
else:
location_ini = args.location_ini
location_ini = os.path.abspath(location_ini)
location_cfg = read_config(filename=location_ini)
# Either use explicit config file location or fallback
# to a file in the same directory as the ichnaea.ini
if not args.alembic_ini:
alembic_ini = os.path.join(os.path.dirname(location_ini),
'alembic.ini')
else:
alembic_ini = args.alembic_ini
alembic_ini = os.path.abspath(alembic_ini)
alembic_cfg = Config(alembic_ini)
alembic_section = alembic_cfg.get_section('alembic')
if _db_rw is None:
db_rw = Database(alembic_section['sqlalchemy.url'])
else:
db_rw = _db_rw
configure_raven(location_cfg.get('ichnaea', 'sentry_dsn'),
_client=_raven_client)
engine = db_rw.engine
create_schema(engine, alembic_cfg, location_cfg)
else:
parser.print_help()
def main(argv, _db_master=None, _heka_client=None):
parser = argparse.ArgumentParser(
prog=argv[0], description='Initialize Ichnaea database')
parser.add_argument('--alembic_ini',
help='Path to the alembic migration config.')
parser.add_argument('--location_ini',
help='Path to the ichnaea app config.')
parser.add_argument('--initdb', action='store_true',
help='Initialize database')
args = parser.parse_args(argv[1:])
if args.initdb:
# Either use explicit config file location or fallback
# on environment variable or finally file in current directory
if not args.location_ini:
location_ini = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini')
else:
location_ini = args.location_ini
location_ini = os.path.abspath(location_ini)
location_cfg = read_config(filename=location_ini)
# Either use explicit config file location or fallback
# to a file in the same directory as the ichnaea.ini
if not args.alembic_ini:
alembic_ini = os.path.join(
os.path.dirname(location_ini), 'alembic.ini')
else:
alembic_ini = args.alembic_ini
alembic_ini = os.path.abspath(alembic_ini)
alembic_cfg = Config(alembic_ini)
alembic_section = alembic_cfg.get_section('alembic')
if _db_master is None:
db_master = Database(alembic_section['sqlalchemy.url'])
else:
db_master = _db_master
configure_heka(location_ini, _heka_client=_heka_client)
engine = db_master.engine
create_schema(engine, alembic_cfg, location_cfg)
else:
parser.print_help()
def main():
settings = read_config().get_map('ichnaea')
db = Database(settings['db_slave'])
session = db.session()
bad = []
offset = 0
count = 10000
results = True
while results:
results = False
r = session.execute("select id, lat, lon, mcc, mnc, lac, cid, radio, "
"total_measures from cell where "
"lat is not null and lon is not null and "
"mcc not in (1, 260) "
"order by id limit %d offset %d" % (count, offset))
offset += count
for row in r:
results = True
(id, lat, lon, mcc, mnc, lac, cid, radio, total_measures) = row
ccs = [c.alpha2 for c in mobile_codes.mcc(str(mcc))]
if not any([location_is_in_country(lat, lon, c, 1) for c in ccs]):
if ccs:
s = ",".join(ccs)
else:
continue
bad.append(
dict(type='Feature',
properties=dict(mcc=mcc,
mnc=mnc,
lac=lac,
cid=cid,
radio=radio,
total_measures=total_measures,
countries=s),
geometry=dict(type='Point', coordinates=[lon, lat])))
json.dump(dict(type='FeatureCollection', features=bad),
sys.stdout,
indent=True)
def main(argv, _db_master=None,
_heka_client=None, _stats_client=None):
# run for example via:
# bin/location_map --create --upload --datamaps=/path/to/datamaps/ \
# --output=ichnaea/content/static/tiles/
parser = argparse.ArgumentParser(
prog=argv[0], description='Generate and upload datamap tiles.')
parser.add_argument('--create', action='store_true',
help='Create tiles.')
parser.add_argument('--upload', action='store_true',
help='Upload tiles to S3.')
parser.add_argument('--concurrency', default=2,
help='How many concurrent render processes to use?')
parser.add_argument('--datamaps',
help='Directory of the datamaps tools.')
parser.add_argument('--output',
help='Optional directory for local tile output.')
args = parser.parse_args(argv[1:])
if args.create:
conf = read_config()
if _db_master:
db = _db_master
else: # pragma: no cover
db = Database(conf.get('ichnaea', 'db_master'))
bucketname = conf.get('ichnaea', 's3_assets_bucket').strip('/')
heka_client = configure_heka(conf.filename, _heka_client=_heka_client)
stats_client = configure_stats(
conf.get('ichnaea', 'statsd_host'), _client=_stats_client)
upload = False
if args.upload: # pragma: no cover
upload = bool(args.upload)
concurrency = 2
if args.concurrency:
concurrency = int(args.concurrency)
datamaps = ''
if args.datamaps:
datamaps = os.path.abspath(args.datamaps)
output = None
if args.output:
output = os.path.abspath(args.output)
try:
with stats_client.timer("datamaps.total_time"):
generate(db, bucketname, heka_client, stats_client,
upload=upload,
concurrency=concurrency,
datamaps=datamaps,
output=output)
except Exception: # pragma: no cover
heka_client.raven(RAVEN_ERROR)
raise
else: # pragma: no cover
parser.print_help()
def _statsd_host():
from ichnaea.app_config import read_config
conf = read_config()
return conf.get_map('ichnaea').get('statsd_host', None)
