def create(location, threads):
""" Create a ListenBrainz data dump which includes a private dump, a statistics dump
and a dump of the actual listens from InfluxDB
Args:
location (str): path to the directory where the dump should be made
threads (int): the number of threads to be used while compression
"""
db.init_db_connection(config.SQLALCHEMY_DATABASE_URI)
ls = init_influx_connection(log, {
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'REDIS_NAMESPACE': config.REDIS_NAMESPACE,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME,
})
time_now = datetime.today()
dump_path = os.path.join(location, 'listenbrainz-dump-{time}'.format(time=time_now.strftime('%Y%m%d-%H%M%S')))
create_path(dump_path)
db_dump.dump_postgres_db(dump_path, time_now, threads)
ls.dump_listens(dump_path, time_now, threads)
try:
write_hashes(dump_path)
except IOError as e:
log.error('Unable to create hash files! Error: %s', str(e))
return
log.info('Dumps created and hashes written at %s' % dump_path)
def init_test_db(force=False):
"""Same as `init_db` command, but creates a database that will be used to
run tests and doesn't import data (no need to do that).
the `PG_CONNECT_TEST` variable must be defined in the config file.
"""
db.init_db_connection(config.POSTGRES_ADMIN_URI)
if force:
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'drop_test_db.sql'))
if not res:
raise Exception(
'Failed to drop existing database and user! Exit code: %i' %
res)
print('Creating user and a database for testing...')
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'create_test_db.sql'))
if not res:
raise Exception(
'Failed to create test user and database! Exit code: %i' % res)
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'))
# Don't raise an exception if the extension already exists
db.engine.dispose()
print("Done!")
def import_musicbrainz_rows(musicbrainz_db_uri, dry_run=True, delete=False):
musicbrainz_db.init_db_engine(musicbrainz_db_uri)
db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI'])
import_count = 0
already_imported = 0
not_found = 0
deleted = 0
if not dry_run:
update_row_ids_for_exceptions()
users = db_user.get_all_users()
with musicbrainz_db.engine.connect() as mb_connection:
with db.engine.connect() as connection:
for user in users:
if user.get('musicbrainz_row_id') is not None:
already_imported += 1
continue
name = user['musicbrainz_id']
result = mb_connection.execute(sqlalchemy.text("""
SELECT id
FROM editor
WHERE LOWER(name) = LOWER(:name)
"""), {
'name': name,
})
musicbrainz_row_id = None
if result.rowcount > 0:
musicbrainz_row_id = result.fetchone()['id']
import_count += 1
else:
print('No user with specified username in the MusicBrainz db: %s' % name)
if delete:
print('Deleting user %s' % name)
try:
delete_user(user)
except NotFound:
print('User %s not found in LB...' % name)
not_found += 1
continue
if not dry_run:
connection.execute(sqlalchemy.text("""
UPDATE "user"
SET musicbrainz_row_id = :musicbrainz_row_id
WHERE id = :id
"""), {
'musicbrainz_row_id': musicbrainz_row_id,
'id': user['id'],
})
print('Inserted row_id %d for user %s' % (musicbrainz_row_id, name))
print('Total number of ListenBrainz users: %d' % len(users))
print('Total number of ListenBrainz users with already imported row ids: %d' % already_imported)
print('Total number of ListenBrainz users whose row ids can be imported: %d' % import_count)
print('Total number of ListenBrainz users not found in MusicBrainz: %d' % not_found)
print('Total number of ListenBrainz users deleted from MusicBrainz: %d' % deleted)
def init_msb_db(force, create_db):
"""Initializes database.
This process involves several steps:
1. Table structure is created.
2. Primary keys and foreign keys are created.
3. Indexes are created.
"""
from listenbrainz import config
db.init_db_connection(config.POSTGRES_ADMIN_URI)
if force:
res = db.run_sql_script_without_transaction(
os.path.join(MSB_ADMIN_SQL_DIR, 'drop_db.sql'))
if not res:
raise Exception(
'Failed to drop existing database and user! Exit code: %s' %
res)
if create_db:
print('Creating user and a database...')
res = db.run_sql_script_without_transaction(
os.path.join(MSB_ADMIN_SQL_DIR, 'create_db.sql'))
if not res:
raise Exception(
'Failed to create new database and user! Exit code: %s' % res)
print('Creating database extensions...')
res = db.run_sql_script_without_transaction(
os.path.join(MSB_ADMIN_SQL_DIR, 'create_extensions.sql'))
# Don't raise an exception if the extension already exists
db.engine.dispose()
# print('Creating schema...')
# exit_code = run_psql_script('create_schema.sql')
# if exit_code != 0:
# raise Exception('Failed to create database schema! Exit code: %i' % exit_code)
db.init_db_connection(config.MESSYBRAINZ_SQLALCHEMY_DATABASE_URI)
print('Creating tables...')
db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_tables.sql'))
print('Creating primary and foreign keys...')
db.run_sql_script(
os.path.join(MSB_ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(
os.path.join(MSB_ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
print('Creating functions...')
db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_functions.sql'))
print('Creating indexes...')
db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def populate_queue(force):
""" Populate the ListenBrainz stats calculation queue with entities
which need their stats calculated.
"""
logger.info('Connecting to database...')
db.init_db_connection(config.SQLALCHEMY_DATABASE_URI)
logger.info('Connected!')
logger.info('Pushing entities whose stats should be calculated into the queue...')
push_entities_to_queue(force=force)
logger.info('Pushed all relevant entities, stats_calculator should calculate stats soon!')
def init_db(force, create_db):
"""Initializes database.
This process involves several steps:
1. Table structure is created.
2. Primary keys and foreign keys are created.
3. Indexes are created.
"""
from listenbrainz import config
db.init_db_connection(config.POSTGRES_ADMIN_URI)
if force:
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'drop_db.sql'))
if not res:
raise Exception(
'Failed to drop existing database and user! Exit code: %i' %
res)
if create_db or force:
print('PG: Creating user and a database...')
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'create_db.sql'))
if not res:
raise Exception(
'Failed to create new database and user! Exit code: %i' % res)
db.init_db_connection(config.POSTGRES_ADMIN_LB_URI)
print('PG: Creating database extensions...')
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'))
# Don't raise an exception if the extension already exists
application = webserver.create_app()
with application.app_context():
print('PG: Creating schema...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_schema.sql'))
print('PG: Creating Types...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_types.sql'))
print('PG: Creating tables...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
print('PG: Creating primary and foreign keys...')
db.run_sql_script(
os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(
os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
print('PG: Creating indexes...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def create_app():
app = Flask(__name__)
# Configuration
from listenbrainz import config
app.config.from_object(config)
# Logging
from listenbrainz.webserver.loggers import init_loggers
init_loggers(app)
# Redis connection
create_redis(app)
# Influx connection
create_influx(app)
# RabbitMQ connection
create_rabbitmq(app)
# Database connection
from listenbrainz import db
db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI'])
from listenbrainz.webserver.external import messybrainz
messybrainz.init_db_connection(
app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI'])
# OAuth
from listenbrainz.webserver.login import login_manager, provider
login_manager.init_app(app)
provider.init(app.config['MUSICBRAINZ_CLIENT_ID'],
app.config['MUSICBRAINZ_CLIENT_SECRET'])
# Error handling
from listenbrainz.webserver.errors import init_error_handlers
init_error_handlers(app)
from listenbrainz.webserver import rate_limiter
@app.after_request
def after_request_callbacks(response):
return rate_limiter.inject_x_rate_headers(response)
# Template utilities
app.jinja_env.add_extension('jinja2.ext.do')
from listenbrainz.webserver import utils
app.jinja_env.filters['date'] = utils.reformat_date
app.jinja_env.filters['datetime'] = utils.reformat_datetime
_register_blueprints(app)
return app
def import_db(location, threads=None):
""" Import a ListenBrainz PostgreSQL dump into the PostgreSQL database.
Note: This method tries to import the private dump first, followed by the statistics
dump. However, in absence of a private dump, it imports sanitized versions of the
user table in the statistics dump in order to satisfy foreign key constraints.
Args:
location (str): path to the directory which contains the private and the stats dump
threads (int): the number of threads to use during decompression, defaults to 1
"""
db.init_db_connection(config.SQLALCHEMY_DATABASE_URI)
db_dump.import_postgres_dump(location, threads)
def populate_queue(force):
""" Populate the ListenBrainz stats calculation queue with entities
which need their stats calculated.
"""
logger.info('Connecting to database...')
db.init_db_connection(config.SQLALCHEMY_DATABASE_URI)
logger.info('Connected!')
logger.info(
'Pushing entities whose stats should be calculated into the queue...')
push_entities_to_queue(force=force)
logger.info(
'Pushed all relevant entities, stats_calculator should calculate stats soon!'
)
def import_dump(private_archive, public_archive, listen_archive, threads):
""" Import a ListenBrainz dump into the database.
Note: This method tries to import the private db dump first, followed by the public db
dump. However, in absence of a private dump, it imports sanitized versions of the
user table in the public dump in order to satisfy foreign key constraints.
Then it imports the listen dump.
Args:
private_archive (str): the path to the ListenBrainz private dump to be imported
public_archive (str): the path to the ListenBrainz public dump to be imported
listen_archive (str): the path to the ListenBrainz listen dump archive to be imported
threads (int): the number of threads to use during decompression, defaults to 1
"""
if not private_archive and not public_archive and not listen_archive:
print('You need to enter a path to the archive(s) to import!')
sys.exit(1)
db.init_db_connection(config.SQLALCHEMY_DATABASE_URI)
db_dump.import_postgres_dump(private_archive, public_archive, threads)
ls = init_influx_connection(
log, {
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'REDIS_NAMESPACE': config.REDIS_NAMESPACE,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME,
})
try:
ls.import_listens_dump(listen_archive, threads)
except IOError as e:
log.error('IOError while trying to import data into Influx: %s',
str(e))
raise
except InfluxDBClientError as e:
log.error('Error while sending data to Influx: %s', str(e))
raise
except InfluxDBServerError as e:
log.error('InfluxDB Server Error while importing data: %s', str(e))
raise
except Exception as e:
log.error('Unexpected error while importing data: %s', str(e))
raise
def init_msb_db(force, create_db):
"""Initializes database.
This process involves several steps:
1. Table structure is created.
2. Primary keys and foreign keys are created.
3. Indexes are created.
"""
db.init_db_connection(config.POSTGRES_ADMIN_URI)
if force:
res = db.run_sql_script_without_transaction(os.path.join(MSB_ADMIN_SQL_DIR, 'drop_db.sql'))
if not res:
raise Exception('Failed to drop existing database and user! Exit code: %s' % res)
if create_db:
print('Creating user and a database...')
res = db.run_sql_script_without_transaction(os.path.join(MSB_ADMIN_SQL_DIR, 'create_db.sql'))
if not res:
raise Exception('Failed to create new database and user! Exit code: %s' % res)
print('Creating database extensions...')
res = db.run_sql_script_without_transaction(os.path.join(MSB_ADMIN_SQL_DIR, 'create_extensions.sql'))
# Don't raise an exception if the extension already exists
db.engine.dispose()
# print('Creating schema...')
# exit_code = run_psql_script('create_schema.sql')
# if exit_code != 0:
# raise Exception('Failed to create database schema! Exit code: %i' % exit_code)
db.init_db_connection(config.MESSYBRAINZ_SQLALCHEMY_DATABASE_URI)
print('Creating tables...')
db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_tables.sql'))
print('Creating primary and foreign keys...')
db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
print('Creating indexes...')
db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def start(self):
""" Starts the stats calculator. This should run perpetually,
monitor the stats rabbitmq queue and calculate stats for
entities that it receives from the queue.
"""
# if no bigquery support, sleep
if not self.app.config['WRITE_TO_BIGQUERY']:
while True:
time.sleep(10000)
self.log.info('Connecting to Google BigQuery...')
stats.init_bigquery_connection()
self.log.info('Connected!')
self.log.info('Connecting to database...')
db.init_db_connection(self.app.config['SQLALCHEMY_DATABASE_URI'])
self.log.info('Connected!')
self.log.info('Connecting to redis...')
self.redis = utils.connect_to_redis(host=self.app.config['REDIS_HOST'],
port=self.app.config['REDIS_PORT'],
log=self.log.error)
self.log.info('Connected!')
while True:
self.init_rabbitmq_connection()
self.incoming_ch = utils.create_channel_to_consume(
connection=self.connection,
exchange=self.app.config['STATS_EXCHANGE'],
queue=self.app.config['STATS_QUEUE'],
callback_function=self.callback,
)
self.log.info('Stats calculator started!')
try:
self.incoming_ch.start_consuming()
except pika.exceptions.ConnectionClosed:
self.log.info("Connection to rabbitmq closed. Re-opening.")
self.connection = None
continue
self.connection.close()
def start(self):
""" Starts the job runner. This should run perpetually,
monitor the bigquery jobs rabbitmq queue and perform tasks for
entries in the queue.
"""
with self.app.app_context():
# if no bigquery support, sleep
if not current_app.config['WRITE_TO_BIGQUERY']:
while True:
time.sleep(10000)
current_app.logger.info('Connecting to Google BigQuery...')
self.bigquery = bigquery.create_bigquery_object()
current_app.logger.info('Connected!')
current_app.logger.info('Connecting to database...')
db.init_db_connection(current_app.config['SQLALCHEMY_DATABASE_URI'])
current_app.logger.info('Connected!')
current_app.logger.info('Connecting to redis...')
self.redis = utils.connect_to_redis(host=current_app.config['REDIS_HOST'], port=current_app.config['REDIS_PORT'], log=current_app.logger.error)
current_app.logger.info('Connected!')
while True:
self.init_rabbitmq_connection()
self.incoming_ch = utils.create_channel_to_consume(
connection=self.connection,
exchange=current_app.config['BIGQUERY_EXCHANGE'],
queue=current_app.config['BIGQUERY_QUEUE'],
callback_function=self.callback,
)
current_app.logger.info('Stats calculator started!')
try:
self.incoming_ch.start_consuming()
except pika.exceptions.ConnectionClosed:
current_app.logger.warning("Connection to rabbitmq closed. Re-opening.")
self.connection = None
continue
self.connection.close()
def init_db(force, create_db):
"""Initializes database.
This process involves several steps:
1. Table structure is created.
2. Primary keys and foreign keys are created.
3. Indexes are created.
"""
db.init_db_connection(config.POSTGRES_ADMIN_URI)
if force:
res = db.run_sql_script_without_transaction(os.path.join(ADMIN_SQL_DIR, 'drop_db.sql'))
if not res:
raise Exception('Failed to drop existing database and user! Exit code: %i' % res)
if create_db:
print('Creating user and a database...')
res = db.run_sql_script_without_transaction(os.path.join(ADMIN_SQL_DIR, 'create_db.sql'))
if not res:
raise Exception('Failed to create new database and user! Exit code: %i' % res)
print('Creating database extensions...')
res = db.run_sql_script_without_transaction(os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'))
# Don't raise an exception if the extension already exists
application = webserver.create_app()
with application.app_context():
print('Creating schema...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_schema.sql'))
print('Creating tables...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
print('Creating primary and foreign keys...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
print('Creating indexes...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def setUp(self):
self.config = config
db.init_db_connection(config.SQLALCHEMY_DATABASE_URI)
self.reset_db()
def import_musicbrainz_rows(musicbrainz_db_uri, dry_run=True, delete=False):
musicbrainz_db.init_db_engine(musicbrainz_db_uri)
db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI'])
import_count = 0
already_imported = 0
not_found = 0
deleted = 0
if not dry_run:
update_row_ids_for_exceptions()
users = db_user.get_all_users()
with musicbrainz_db.engine.connect() as mb_connection:
with db.engine.connect() as connection:
for user in users:
if user.get('musicbrainz_row_id') is not None:
already_imported += 1
continue
name = user['musicbrainz_id']
result = mb_connection.execute(
sqlalchemy.text("""
SELECT id
FROM editor
WHERE LOWER(name) = LOWER(:name)
"""), {
'name': name,
})
musicbrainz_row_id = None
if result.rowcount > 0:
musicbrainz_row_id = result.fetchone()['id']
import_count += 1
else:
print(
'No user with specified username in the MusicBrainz db: %s'
% name)
if delete:
print('Deleting user %s' % name)
try:
delete_user(user)
except NotFound:
print('User %s not found in LB...' % name)
not_found += 1
continue
if not dry_run:
connection.execute(
sqlalchemy.text("""
UPDATE "user"
SET musicbrainz_row_id = :musicbrainz_row_id
WHERE id = :id
"""), {
'musicbrainz_row_id': musicbrainz_row_id,
'id': user['id'],
})
print('Inserted row_id %d for user %s' %
(musicbrainz_row_id, name))
print('Total number of ListenBrainz users: %d' % len(users))
print(
'Total number of ListenBrainz users with already imported row ids: %d'
% already_imported)
print(
'Total number of ListenBrainz users whose row ids can be imported: %d'
% import_count)
print('Total number of ListenBrainz users not found in MusicBrainz: %d' %
not_found)
print('Total number of ListenBrainz users deleted from MusicBrainz: %d' %
deleted)
def gen_app(config_path=None, debug=None):
""" Generate a Flask app for LB with all configurations done and connections established.
In the Flask app returned, blueprints are not registered.
"""
app = CustomFlask(
import_name=__name__,
use_flask_uuid=True,
)
print("Starting metabrainz service with %s environment." % deploy_env)
# Load configuration files: If we're running under a docker deployment, wait until
config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'..', 'config.py')
if deploy_env:
print("Checking if consul template generated config file exists: %s" %
config_file)
for i in range(CONSUL_CONFIG_FILE_RETRY_COUNT):
if not os.path.exists(config_file):
sleep(1)
if not os.path.exists(config_file):
print(
"No configuration file generated yet. Retried %d times, exiting."
% CONSUL_CONFIG_FILE_RETRY_COUNT)
sys.exit(-1)
print("loading consul config file %s)" % config_file)
app.config.from_pyfile(config_file)
else:
app.config.from_pyfile(config_file)
if debug is not None:
app.debug = debug
# initialize Flask-DebugToolbar if the debug option is True
if app.debug and app.config['SECRET_KEY']:
app.init_debug_toolbar()
# Output config values and some other info
print('Configuration values are as follows: ')
print(pprint.pformat(app.config, indent=4))
try:
with open('.git-version') as git_version_file:
print('Running on git commit: %s', git_version_file.read().strip())
except IOError as e:
print('Unable to retrieve git commit. Error: %s', str(e))
# Logging
app.init_loggers(file_config=app.config.get('LOG_FILE'),
email_config=app.config.get('LOG_EMAIL'),
sentry_config=app.config.get('LOG_SENTRY'))
# Redis connection
create_redis(app)
# Influx connection
create_influx(app)
# RabbitMQ connection
create_rabbitmq(app)
# Database connection
from listenbrainz import db
db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI'])
from listenbrainz.webserver.external import messybrainz
messybrainz.init_db_connection(
app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI'])
# OAuth
from listenbrainz.webserver.login import login_manager, provider
login_manager.init_app(app)
provider.init(app.config['MUSICBRAINZ_CLIENT_ID'],
app.config['MUSICBRAINZ_CLIENT_SECRET'])
# Error handling
from listenbrainz.webserver.errors import init_error_handlers
init_error_handlers(app)
from listenbrainz.webserver import rate_limiter
@app.after_request
def after_request_callbacks(response):
return rate_limiter.inject_x_rate_headers(response)
# Template utilities
app.jinja_env.add_extension('jinja2.ext.do')
from listenbrainz.webserver import utils
app.jinja_env.filters['date'] = utils.reformat_date
app.jinja_env.filters['datetime'] = utils.reformat_datetime
return app
def gen_app(config_path=None, debug=None):
""" Generate a Flask app for LB with all configurations done and connections established.
In the Flask app returned, blueprints are not registered.
"""
app = CustomFlask(
import_name=__name__,
use_flask_uuid=True,
)
load_config(app)
if debug is not None:
app.debug = debug
# initialize Flask-DebugToolbar if the debug option is True
if app.debug and app.config['SECRET_KEY']:
app.init_debug_toolbar()
# Logging
app.init_loggers(file_config=app.config.get('LOG_FILE'),
email_config=app.config.get('LOG_EMAIL'),
sentry_config=app.config.get('LOG_SENTRY'))
# Redis connection
create_redis(app)
# Influx connection
create_influx(app)
# RabbitMQ connection
try:
create_rabbitmq(app)
except ConnectionError:
app.logger.critical("RabbitMQ service is not up!", exc_info=True)
# Database connection
from listenbrainz import db
db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI'])
from listenbrainz.webserver.external import messybrainz
messybrainz.init_db_connection(
app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI'])
if app.config['MB_DATABASE_URI']:
from brainzutils import musicbrainz_db
musicbrainz_db.init_db_engine(app.config['MB_DATABASE_URI'])
# OAuth
from listenbrainz.webserver.login import login_manager, provider
login_manager.init_app(app)
provider.init(app.config['MUSICBRAINZ_CLIENT_ID'],
app.config['MUSICBRAINZ_CLIENT_SECRET'])
# Error handling
from listenbrainz.webserver.errors import init_error_handlers
init_error_handlers(app)
from listenbrainz.webserver import rate_limiter
@app.after_request
def after_request_callbacks(response):
return rate_limiter.inject_x_rate_headers(response)
# Template utilities
app.jinja_env.add_extension('jinja2.ext.do')
from listenbrainz.webserver import utils
app.jinja_env.filters['date'] = utils.reformat_date
app.jinja_env.filters['datetime'] = utils.reformat_datetime
return app
def recalculate_all_user_data():
timescale.init_db_connection(config.SQLALCHEMY_TIMESCALE_URI)
db.init_db_connection(config.SQLALCHEMY_DATABASE_URI)
init_cache(host=config.REDIS_HOST,
port=config.REDIS_PORT,
namespace=config.REDIS_NAMESPACE)
# Find the created timestamp of the last listen
query = "SELECT max(created) FROM listen WHERE created > :date"
try:
with timescale.engine.connect() as connection:
result = connection.execute(sqlalchemy.text(query),
date=datetime.now() -
timedelta(weeks=4))
row = result.fetchone()
last_created_ts = row[0]
except psycopg2.OperationalError as e:
logger.error("Cannot query ts to fetch latest listen." % str(e),
exc_info=True)
raise
logger.info("Last created timestamp: " + str(last_created_ts))
# Select a list of users
user_list = []
query = 'SELECT musicbrainz_id FROM "user"'
try:
with db.engine.connect() as connection:
result = connection.execute(sqlalchemy.text(query))
for row in result:
user_list.append(row[0])
except psycopg2.OperationalError as e:
logger.error("Cannot query db to fetch user list." % str(e),
exc_info=True)
raise
logger.info("Fetched %d users. Setting empty cache entries." %
len(user_list))
# Reset the timestamps and listen counts to 0 for all users
for user_name in user_list:
cache.set(REDIS_USER_LISTEN_COUNT + user_name,
0,
expirein=0,
encode=False)
cache.set(REDIS_USER_LISTEN_COUNT + user_name,
0,
expirein=0,
encode=False)
cache.set(REDIS_USER_TIMESTAMPS + user_name, "0,0", expirein=0)
# Tabulate all of the listen counts/timestamps for all users
logger.info("Scan the whole listen table...")
listen_counts = defaultdict(int)
user_timestamps = {}
query = "SELECT listened_at, user_name FROM listen where created <= :ts"
try:
with timescale.engine.connect() as connection:
result = connection.execute(sqlalchemy.text(query),
ts=last_created_ts)
for row in result:
ts = row[0]
user_name = row[1]
if user_name not in user_timestamps:
user_timestamps[user_name] = [ts, ts]
else:
if ts > user_timestamps[user_name][1]:
user_timestamps[user_name][1] = ts
if ts < user_timestamps[user_name][0]:
user_timestamps[user_name][0] = ts
listen_counts[user_name] += 1
except psycopg2.OperationalError as e:
logger.error("Cannot query db to fetch user list." % str(e),
exc_info=True)
raise
logger.info("Setting updated cache entries.")
# Set the timestamps and listen counts for all users
for user_name in user_list:
try:
cache.increment(REDIS_USER_LISTEN_COUNT + user_name,
amount=listen_counts[user_name])
except KeyError:
pass
try:
tss = cache.get(REDIS_USER_TIMESTAMPS + user_name)
(min_ts, max_ts) = tss.split(",")
min_ts = int(min_ts)
max_ts = int(max_ts)
if min_ts and min_ts < user_timestamps[user_name][0]:
user_timestamps[user_name][0] = min_ts
if max_ts and max_ts > user_timestamps[user_name][1]:
user_timestamps[user_name][1] = max_ts
cache.set(
REDIS_USER_TIMESTAMPS + user_name,
"%d,%d" %
(user_timestamps[user_name][0], user_timestamps[user_name][1]),
expirein=0)
except KeyError:
pass
def create_app(debug=None):
""" Generate a Flask app for LB with all configurations done and connections established.
In the Flask app returned, blueprints are not registered.
"""
app = CustomFlask(
import_name=__name__,
use_flask_uuid=True,
)
load_config(app)
if debug is not None:
app.debug = debug
# As early as possible, if debug is True, set the log level of our 'listenbrainz' logger to DEBUG
# to prevent flask from creating a new log handler
if app.debug:
logger = logging.getLogger('listenbrainz')
logger.setLevel(logging.DEBUG)
# initialize Flask-DebugToolbar if the debug option is True
if app.debug and app.config['SECRET_KEY']:
app.init_debug_toolbar()
sentry_config = app.config.get('LOG_SENTRY')
if sentry_config:
sentry.init_sentry(**sentry_config)
# Initialize BU cache and metrics
cache.init(host=app.config['REDIS_HOST'],
port=app.config['REDIS_PORT'],
namespace=app.config['REDIS_NAMESPACE'])
metrics.init("listenbrainz")
# Database connections
from listenbrainz import db
from listenbrainz.db import timescale as ts
from listenbrainz import messybrainz as msb
db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI'])
ts.init_db_connection(app.config['SQLALCHEMY_TIMESCALE_URI'])
msb.init_db_connection(app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI'])
# Redis connection
from listenbrainz.webserver.redis_connection import init_redis_connection
init_redis_connection(app.logger)
# Timescale connection
from listenbrainz.webserver.timescale_connection import init_timescale_connection
init_timescale_connection(app)
# RabbitMQ connection
from listenbrainz.webserver.rabbitmq_connection import init_rabbitmq_connection
try:
init_rabbitmq_connection(app)
except ConnectionError:
app.logger.critical("RabbitMQ service is not up!", exc_info=True)
if app.config['MB_DATABASE_URI']:
from brainzutils import musicbrainz_db
musicbrainz_db.init_db_engine(app.config['MB_DATABASE_URI'])
# OAuth
from listenbrainz.webserver.login import login_manager, provider
login_manager.init_app(app)
provider.init(app.config['MUSICBRAINZ_CLIENT_ID'],
app.config['MUSICBRAINZ_CLIENT_SECRET'])
# Error handling
from listenbrainz.webserver.errors import init_error_handlers
init_error_handlers(app)
from brainzutils.ratelimit import inject_x_rate_headers, set_user_validation_function
set_user_validation_function(check_ratelimit_token_whitelist)
@app.after_request
def after_request_callbacks(response):
return inject_x_rate_headers(response)
# Template utilities
app.jinja_env.add_extension('jinja2.ext.do')
from listenbrainz.webserver import utils
app.jinja_env.filters['date'] = utils.reformat_date
app.jinja_env.filters['datetime'] = utils.reformat_datetime
return app
def setUp(self):
self.config = config
db.init_db_connection(config.SQLALCHEMY_DATABASE_URI)
self.reset_db()
