def init_test_db(force=False):
"""Same as `init_db` command, but creates a database that will be used to
run tests and doesn't import data (no need to do that).
the `PG_CONNECT_TEST` variable must be defined in the config file.
"""
db.init_db_connection(config.POSTGRES_ADMIN_URI)
if force:
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'drop_test_db.sql'))
if not res:
raise Exception(
'Failed to drop existing database and user! Exit code: %i' %
res)
print('Creating user and a database for testing...')
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'create_test_db.sql'))
if not res:
raise Exception(
'Failed to create test user and database! Exit code: %i' % res)
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'))
# Don't raise an exception if the extension already exists
db.engine.dispose()
print("Done!")
def init_test_db(force=False):
"""Same as `init_db` command, but creates a database that will be used to
run tests and doesn't import data (no need to do that).
`PG_CONNECT_TEST` variable must be defined in the config file.
"""
if force:
exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' < ' +
os.path.join(ADMIN_SQL_DIR, 'drop_test_db.sql'),
shell=True)
if exit_code != 0:
raise Exception('Failed to drop existing database and user! Exit code: %i' % exit_code)
print('Creating database and user for testing...')
exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' < ' +
os.path.join(ADMIN_SQL_DIR, 'create_test_db.sql'),
shell=True)
if exit_code != 0:
raise Exception('Failed to create new database and user! Exit code: %i' % exit_code)
exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' -d ab_test < ' +
os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'),
shell=True)
if exit_code != 0:
raise Exception('Failed to create database extensions! Exit code: %i' % exit_code)
db.init_db_connection(config.PG_CONNECT_TEST)
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_types.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def create_app():
app = Flask(__name__)
# Configuration
sys.path.append(
os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))
sys.path.append(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
"../listenstore"))
import config
app.config.from_object(config)
# Logging
from webserver.loggers import init_loggers
init_loggers(app)
# Redis connection
create_redis(app)
# Postgres connection
create_postgres(app)
# Influx connection
create_influx(app)
# Database connection
import db
db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI'])
from webserver.external import messybrainz
messybrainz.init_db_connection(
app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI'])
# OAuth
from webserver.login import login_manager, provider
login_manager.init_app(app)
provider.init(app.config['MUSICBRAINZ_CLIENT_ID'],
app.config['MUSICBRAINZ_CLIENT_SECRET'])
# Error handling
from webserver.errors import init_error_handlers
init_error_handlers(app)
from webserver import rate_limiter
@app.after_request
def after_request_callbacks(response):
return rate_limiter.inject_x_rate_headers(response)
# Template utilities
app.jinja_env.add_extension('jinja2.ext.do')
from webserver import utils
app.jinja_env.filters['date'] = utils.reformat_date
app.jinja_env.filters['datetime'] = utils.reformat_datetime
_register_blueprints(app)
return app
def main():
logging.info("Starting dataset evaluator...")
while True:
db.init_db_connection(config.PG_CONNECT)
pending_job = db.dataset_eval.get_next_pending_job()
if pending_job:
logging.info("Processing job %s..." % pending_job["id"])
evaluate_dataset(pending_job)
else:
logging.info("No pending datasets. Sleeping %s seconds." % SLEEP_DURATION)
db.close_connection()
time.sleep(SLEEP_DURATION)
def init_msb_db(force, create_db):
"""Initializes database.
This process involves several steps:
1. Table structure is created.
2. Primary keys and foreign keys are created.
3. Indexes are created.
"""
db.init_db_connection(config.POSTGRES_ADMIN_URI)
if force:
res = db.run_sql_script_without_transaction(
os.path.join(MSB_ADMIN_SQL_DIR, 'drop_db.sql'))
if not res:
raise Exception(
'Failed to drop existing database and user! Exit code: %s' %
res)
if create_db:
print('Creating user and a database...')
res = db.run_sql_script_without_transaction(
os.path.join(MSB_ADMIN_SQL_DIR, 'create_db.sql'))
if not res:
raise Exception(
'Failed to create new database and user! Exit code: %s' % res)
print('Creating database extensions...')
res = db.run_sql_script_without_transaction(
os.path.join(MSB_ADMIN_SQL_DIR, 'create_extensions.sql'))
# Don't raise an exception if the extension already exists
db.engine.dispose()
# print('Creating schema...')
# exit_code = run_psql_script('create_schema.sql')
# if exit_code != 0:
# raise Exception('Failed to create database schema! Exit code: %i' % exit_code)
db.init_db_connection(config.MESSYBRAINZ_SQLALCHEMY_DATABASE_URI)
print('Creating tables...')
db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_tables.sql'))
print('Creating primary and foreign keys...')
db.run_sql_script(
os.path.join(MSB_ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(
os.path.join(MSB_ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
print('Creating indexes...')
db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def init_db(archive, force):
"""Initializes database and imports data if needed.
This process involves several steps:
1. Table structure is created.
2. Data is imported from the archive if it is specified.
3. Primary keys and foreign keys are created.
4. Indexes are created.
Data dump needs to be a .tar.xz archive produced by export command.
More information about populating a PostgreSQL database efficiently can be
found at http://www.postgresql.org/docs/current/static/populate.html.
"""
if force:
exit_code = _run_psql('drop_db.sql')
if exit_code != 0:
raise Exception('Failed to drop existing database and user! Exit code: %i' % exit_code)
print('Creating user and a database...')
exit_code = _run_psql('create_db.sql')
if exit_code != 0:
raise Exception('Failed to create new database and user! Exit code: %i' % exit_code)
print('Creating database extensions...')
exit_code = _run_psql('create_extensions.sql', 'acousticbrainz')
if exit_code != 0:
raise Exception('Failed to create database extensions! Exit code: %i' % exit_code)
db.init_db_connection(config.PG_CONNECT)
print('Creating types...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_types.sql'))
print('Creating tables...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
if archive:
print('Importing data...')
db.dump.import_db_dump(archive)
else:
print('Skipping data importing.')
print('Creating primary and foreign keys...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
print('Creating indexes...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def init_db(force, create_db):
"""Initializes database.
This process involves several steps:
1. Table structure is created.
2. Primary keys and foreign keys are created.
3. Indexes are created.
"""
db.init_db_connection(config.POSTGRES_ADMIN_URI)
if force:
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'drop_db.sql'))
if not res:
raise Exception(
'Failed to drop existing database and user! Exit code: %i' %
res)
if create_db:
print('Creating user and a database...')
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'create_db.sql'))
if not res:
raise Exception(
'Failed to create new database and user! Exit code: %i' % res)
print('Creating database extensions...')
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'))
# Don't raise an exception if the extension already exists
application = webserver.create_app()
with application.app_context():
print('Creating schema...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_schema.sql'))
print('Creating tables...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
print('Creating primary and foreign keys...')
db.run_sql_script(
os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(
os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
print('Creating indexes...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print('Create influx database...')
subprocess.call(os.path.join(ADMIN_INFLUX_DIR, 'create_db.py'))
print("Done!")
def create_app():
global _kafka
app = Flask(__name__)
# Configuration
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../listenstore"))
import config
app.config.from_object(config)
# Logging
from webserver.loggers import init_loggers
init_loggers(app)
# Kafka connection
from kafka_connection import init_kafka_connection
init_kafka_connection(app.config['KAFKA_CONNECT'])
# Database connection
from db import init_db_connection
init_db_connection(app)
messybrainz.db.init_db_engine(app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI'])
# OAuth
from webserver.login import login_manager, provider
login_manager.init_app(app)
provider.init(app.config['MUSICBRAINZ_CLIENT_ID'],
app.config['MUSICBRAINZ_CLIENT_SECRET'])
# Error handling
from webserver.errors import init_error_handlers
init_error_handlers(app)
# Template utilities
app.jinja_env.add_extension('jinja2.ext.do')
from webserver import utils
app.jinja_env.filters['date'] = utils.reformat_date
app.jinja_env.filters['datetime'] = utils.reformat_datetime
# Blueprints
from webserver.views.index import index_bp
from webserver.views.login import login_bp
from webserver.views.api import api_bp
from webserver.views.user import user_bp
app.register_blueprint(index_bp)
app.register_blueprint(login_bp, url_prefix='/login')
app.register_blueprint(user_bp, url_prefix='/user')
app.register_blueprint(api_bp)
return app
def create_app():
global _kafka
app = Flask(__name__)
# Configuration
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../listenstore"))
import config
app.config.from_object(config)
# Logging
from webserver.loggers import init_loggers
init_loggers(app)
# Kafka connection
from kafka_connection import init_kafka_connection
init_kafka_connection(app.config['KAFKA_CONNECT'])
# Database connection
from db import init_db_connection
init_db_connection(app)
# OAuth
from webserver.login import login_manager, provider
login_manager.init_app(app)
provider.init(app.config['MUSICBRAINZ_CLIENT_ID'],
app.config['MUSICBRAINZ_CLIENT_SECRET'])
# Error handling
from webserver.errors import init_error_handlers
init_error_handlers(app)
# Template utilities
app.jinja_env.add_extension('jinja2.ext.do')
from webserver import utils
app.jinja_env.filters['date'] = utils.reformat_date
app.jinja_env.filters['datetime'] = utils.reformat_datetime
# Blueprints
from webserver.views.index import index_bp
from webserver.views.login import login_bp
from webserver.views.api import api_bp
from webserver.views.user import user_bp
app.register_blueprint(index_bp)
app.register_blueprint(login_bp, url_prefix='/login')
app.register_blueprint(user_bp, url_prefix='/user')
app.register_blueprint(api_bp)
return app
def create_app():
app = Flask(__name__)
# Configuration
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../listenstore"))
import config
app.config.from_object(config)
# Logging
from webserver.loggers import init_loggers
init_loggers(app)
# Redis connection (RedisStore)
create_redis(app)
# Postgres connection (ListenStore)
create_postgres(app)
# Database connection
import db
db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI'])
from webserver.external import messybrainz
messybrainz.init_db_connection(app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI'])
# OAuth
from webserver.login import login_manager, provider
login_manager.init_app(app)
provider.init(app.config['MUSICBRAINZ_CLIENT_ID'],
app.config['MUSICBRAINZ_CLIENT_SECRET'])
# Error handling
from webserver.errors import init_error_handlers
init_error_handlers(app)
from webserver import rate_limiter
@app.after_request
def after_request_callbacks(response):
return rate_limiter.inject_x_rate_headers(response)
# Template utilities
app.jinja_env.add_extension('jinja2.ext.do')
from webserver import utils
app.jinja_env.filters['date'] = utils.reformat_date
app.jinja_env.filters['datetime'] = utils.reformat_datetime
_register_blueprints(app)
return app
def setUp(self):
init_db_connection(config.PG_CONNECT_TEST)
self.reset_db()
def import_data(archive):
"""Imports data dump into the database."""
db.init_db_connection(config.PG_CONNECT)
print('Importing data...')
db.dump.import_db_dump(archive)
def import_data(archive):
"""Imports data dump into the database."""
db.init_db_connection(config.PG_CONNECT)
print('Importing data...')
db.dump.import_db_dump(archive)
def runserver(host, port, debug):
db.init_db_connection(config.PG_CONNECT)
create_app().run(host=host, port=port, debug=debug)
def setUp(self):
self.config = config
db.init_db_connection(config.SQLALCHEMY_DATABASE_URI)
self.reset_db()
def create_app():
app = Flask(__name__)
# Configuration
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))
import config
app.config.from_object(config)
# Logging
from webserver.loggers import init_loggers
init_loggers(app)
# Database connection
from db import init_db_connection
init_db_connection(app.config['PG_CONNECT'])
# Memcached
if 'MEMCACHED_SERVERS' in app.config:
from db import cache
cache.init(app.config['MEMCACHED_SERVERS'],
app.config['MEMCACHED_NAMESPACE'],
debug=1 if app.debug else 0)
# Extensions
from flask_uuid import FlaskUUID
FlaskUUID(app)
# MusicBrainz
import musicbrainzngs
from db import SCHEMA_VERSION
musicbrainzngs.set_useragent(app.config['MUSICBRAINZ_USERAGENT'], SCHEMA_VERSION)
if app.config['MUSICBRAINZ_HOSTNAME']:
musicbrainzngs.set_hostname(app.config['MUSICBRAINZ_HOSTNAME'])
# OAuth
from webserver.login import login_manager, provider
login_manager.init_app(app)
provider.init(app.config['MUSICBRAINZ_CLIENT_ID'],
app.config['MUSICBRAINZ_CLIENT_SECRET'])
# Error handling
from webserver.errors import init_error_handlers
init_error_handlers(app)
# Template utilities
app.jinja_env.add_extension('jinja2.ext.do')
from webserver import utils
app.jinja_env.filters['date'] = utils.reformat_date
app.jinja_env.filters['datetime'] = utils.reformat_datetime
# Blueprints
from webserver.views.index import index_bp
from webserver.views.data import data_bp
from webserver.views.api import api_bp
from webserver.views.stats import stats_bp
from webserver.views.login import login_bp
from webserver.views.user import user_bp
from webserver.views.datasets import datasets_bp
app.register_blueprint(index_bp)
app.register_blueprint(data_bp)
app.register_blueprint(api_bp)
app.register_blueprint(stats_bp)
app.register_blueprint(login_bp, url_prefix='/login')
app.register_blueprint(user_bp, url_prefix='/user')
app.register_blueprint(datasets_bp, url_prefix='/datasets')
return app
def setUp(self):
self.config = config
db.init_db_connection(config.TEST_SQLALCHEMY_DATABASE_URI)
self.reset_db()
def setUp(self):
init_db_connection(config.PG_CONNECT_TEST)
self.reset_db()