def build():
logger.info('Dropping and recreating database')
db_url_base, db_name = settings.DATABASE_URL.rsplit('/', 1)
with psycopg2.connect(db_url_base + '/postgres') as conn:
with conn.cursor() as cur:
conn.set_isolation_level(0)
cur.execute('DROP DATABASE IF EXISTS %s' % (db_name,))
cur.execute('CREATE DATABASE %s' % (db_name,))
db.migrate()
file_infos = tasks.poll()
# Make sure we finish parsing MOSMIX before going ahead as current
# observations depend on it
tasks.parse(url=next(file_infos)['url'], export=True)
with ThreadPoolExecutor(max_workers=2*cpu_count()+1) as executor:
with _time('Database creation time', unit='h'):
futures = [
executor.submit(tasks.parse, url=file_info['url'], export=True)
for file_info in file_infos]
finished, pending = wait(futures, return_when=FIRST_EXCEPTION)
for f in pending:
f.cancel()
for f in finished:
# Re-raise any occured exceptions
if exc := f.exception():
raise exc
def migrate():
"""Apply all pending database migrations."""
db.migrate()
def migrate_callback(ctx, param, value):
if value:
db.migrate()
def migrate():
db.migrate()