def fetch_and_store_releases(verbose=False):
""" Fetches releases from the musicbrainz database for the recording MBIDs
in the recording_json table submitted while submitting a listen. It fetches
only the releases for the recordings MBIDs which are not in recording_release_join
table. In the end it prints to the console the total recording MBIDs it processed
and the total recording MBIDs it added to the recording_release_join table.
"""
print("Fetching release for recording MBIDs...")
if verbose:
logging.basicConfig(format='%(message)s', level=logging.DEBUG)
# Init databases
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
musicbrainz_db.init_db_engine(config.MB_DATABASE_URI)
try:
logging.debug("=" * 80)
num_recording_mbids_processed, num_recording_mbids_added = release.fetch_and_store_releases_for_all_recording_mbids()
logging.debug("=" * 80)
print("Total recording MBIDs processed: {0}.".format(num_recording_mbids_processed))
print("Total recording MBIDs added to table: {0}.".format(num_recording_mbids_added))
print("Done!")
except Exception as error:
print("Unable to fetch releases. An error occured: {0}".format(error))
raise
def init_test_db(force=False):
"""Same as `init_db` command, but creates a database that will be used to run tests.
`TEST_SQLALCHEMY_DATABASE_URI` variable must be defined in the config file.
"""
if force:
exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' < ' +
os.path.join(ADMIN_SQL_DIR, 'drop_test_db.sql'),
shell=True)
if exit_code != 0:
raise Exception('Failed to drop existing database and user! Exit code: %i' % exit_code)
print('Creating database and user for testing...')
exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' < ' +
os.path.join(ADMIN_SQL_DIR, 'create_test_db.sql'),
shell=True)
if exit_code != 0:
raise Exception('Failed to create new database and user! Exit code: %i' % exit_code)
exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' -d msb_test < ' +
os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'),
shell=True)
if exit_code != 0:
raise Exception('Failed to create database extensions! Exit code: %i' % exit_code)
db.init_db_engine(config.TEST_SQLALCHEMY_DATABASE_URI)
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def create_artist_credit_clusters_for_mbids(verbose='WARNING'):
"""Creates clusters for artist_credits using artist MBIDs present in
recording_json table.
"""
try:
if verbose == 'INFO':
logging.basicConfig(format='%(message)s', level=logging.INFO)
elif verbose == 'DEBUG':
logging.basicConfig(format='%(message)s', level=logging.DEBUG)
else:
print("Invalid logging level specified. Using default logging level(WARNING).")
print("Creating artist_credit clusters...")
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
logging.debug("=" * 80)
clusters_modified, clusters_add_to_redirect = create_artist_credit_clusters()
logging.debug("=" * 80)
print("Clusters modified: {0}.".format(clusters_modified))
print("Clusters add to redirect table: {0}.".format(clusters_add_to_redirect))
print("Done!")
except Exception as error:
print("While creating artist_credit clusters. An error occured: {0}".format(error))
raise
def init_test_db(force=False):
"""Same as `init_db` command, but creates a database that will be used to run tests.
`TEST_SQLALCHEMY_DATABASE_URI` variable must be defined in the config file.
"""
db.init_db_engine(config.POSTGRES_ADMIN_URI)
if force:
exit_code = db.run_sql_script_without_transaction(os.path.join(ADMIN_SQL_DIR, 'drop_test_db.sql'))
if not exit_code:
raise Exception('Failed to drop existing database and user! Exit code: %i' % exit_code)
print('Creating database and user for testing...')
exit_code = db.run_sql_script_without_transaction(os.path.join(ADMIN_SQL_DIR, 'create_test_db.sql'))
if not exit_code:
raise Exception('Failed to create new database and user! Exit code: %i' % exit_code)
exit_code = db.run_sql_script_without_transaction(os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'))
if not exit_code:
raise Exception('Failed to create database extensions! Exit code: %i' % exit_code)
db.init_db_engine(config.TEST_SQLALCHEMY_DATABASE_URI)
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_functions.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def truncate_release_cluster_and_redirect():
"""Truncate release_cluster and release_redirect tables."""
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
try:
release.truncate_release_cluster_and_release_redirect_table()
print("release_cluster and release_redirect table truncated.")
except Exception as error:
print("An error occured while truncating release_cluster and release_redirect: {0}".format(error))
raise
def truncate_recording_release_join_table():
"""Truncate table recording_release_join."""
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
try:
release.truncate_recording_release_join()
print("Table recording_release_join truncated.")
except Exception as error:
print("An error occured while truncating recording_release_join table: {0}".format(error))
raise
def create_recording_clusters_for_mbids():
"""Creates clusters for recording using recording MBIDs present in
recording_json table.
"""
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
try:
clusters_modified, clusters_add_to_redirect = create_recording_clusters()
print("Clusters modified: {0}.".format(clusters_modified))
print("Clusters add to redirect table: {0}.".format(clusters_add_to_redirect))
print ("Done!")
except Exception as error:
print("While creating recording clusters. An error occured: {0}".format(error))
raise
def truncate_artist_credit_cluster_and_redirect():
"""Truncate artist_credit_cluster and artist_credit_redirect table."""
logging.basicConfig(format='%(message)s', level=logging.INFO)
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
try:
truncate_artist_credit_cluster_and_redirect_tables()
logging.info("artist_credit_cluster and artist_credit_redirect table truncated.")
except Exception as error:
logging.error("An error occured while truncating artist_credit_cluster"
"and artist_credit_redirect table: {0}".format(error)
)
raise
def init_db(force):
"""Initializes database.
This process involves several steps:
1. Table structure is created.
2. Primary keys and foreign keys are created.
3. Indexes are created.
"""
if force:
exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' < ' +
os.path.join(ADMIN_SQL_DIR, 'drop_db.sql'),
shell=True)
if exit_code != 0:
raise Exception('Failed to drop existing database and user! Exit code: %i' % exit_code)
print('Creating user and a database...')
exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' < ' +
os.path.join(ADMIN_SQL_DIR, 'create_db.sql'),
shell=True)
if exit_code != 0:
raise Exception('Failed to create new database and user! Exit code: %i' % exit_code)
print('Creating database extensions...')
exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' -d messybrainz < ' +
os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'),
shell=True)
if exit_code != 0:
raise Exception('Failed to create database extensions! Exit code: %i' % exit_code)
app = create_app()
with app.app_context():
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
print('Creating tables...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
print('Creating primary and foreign keys...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
print('Creating indexes...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def init_db(force):
"""Initializes database.
This process involves several steps:
1. Table structure is created.
2. Primary keys and foreign keys are created.
3. Indexes are created.
"""
db.init_db_engine(config.POSTGRES_ADMIN_URI)
if force:
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'drop_db.sql'))
if not res:
raise Exception(
'Failed to drop existing database and user! Exit code: %i' %
exit_code)
print('Creating user and a database...')
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'create_db.sql'))
if not res:
raise Exception(
'Failed to create new database and user! Exit code: %i' %
exit_code)
print('Creating database extensions...')
exit_code = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'))
app = create_app()
with app.app_context():
print('Creating tables...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
print('Creating primary and foreign keys...')
db.run_sql_script(
os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(
os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
print('Creating indexes...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def fetch_and_store_artist_mbids():
""" Fetches artist MBIDs from the musicbrainz database for the recording MBIDs
in the recording_json table submitted while submitting a listen. It fetches
only the artist MBIDs for the recordings MBIDs which are not in recording_artist_join
table. In the end it prints to the console the total recording MBIDs it processed
and the total recording MBIDs it added to the recording_artist_join table.
"""
# Init databases
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
musicbrainz_db.init_db_engine(config.MB_DATABASE_URI)
try:
num_recording_mbids_processed, num_recording_mbids_added = fetch_and_store_artist_mbids_for_all_recording_mbids()
print("Total recording MBIDs processed: {0}.".format(num_recording_mbids_processed))
print("Total recording MBIDs added to table: {0}.".format(num_recording_mbids_added))
print("Done!")
except Exception as error:
print("Unable to fetch artist MBIDs. An error occured: {0}".format(error))
raise
def create_release_clusters_for_mbids(verbose=0):
"""Creates clusters for release using release MBIDs present in
recording_json table.
"""
if verbose == 1:
logging.basicConfig(format='%(message)s', level=logging.INFO)
elif verbose == 2:
logging.basicConfig(format='%(message)s', level=logging.DEBUG)
print("Creating release clusters...")
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
try:
logging.info("=" * 80)
clusters_modified, clusters_add_to_redirect = release.create_release_clusters()
logging.info("=" * 80)
print("Clusters modified: {0}.".format(clusters_modified))
print("Clusters add to redirect table: {0}.".format(clusters_add_to_redirect))
print ("Done!")
except Exception as error:
print("While creating release clusters. An error occured: {0}".format(error))
raise
def create_clusters_using_fetched_artist_mbids(verbose="WARNING"):
"""Creates clusters for artist_credits using artist MBIDs fetched from MusicBrainz
database and stored in recording_artist_join table.
"""
try:
if verbose == "INFO":
logging.basicConfig(format='%(message)s', level=logging.INFO)
elif verbose == "DEBUG":
logging.basicConfig(format='%(message)s', level=logging.DEBUG)
elif verbose != "WARNING":
print("Invalid logging level specified. Using default logging level(WARNING).")
print("Creating artist_credit clusters...")
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
logging.debug("=" * 80)
clusters_modified, clusters_add_to_redirect = artist.create_clusters_using_fetched_artist_mbids()
logging.debug("=" * 80)
print("Clusters modified: {0}.".format(clusters_modified))
print("Clusters add to redirect table: {0}.".format(clusters_add_to_redirect))
print("Done!")
except Exception as error:
print("While creating artist_credit clusters using fetched artist MBIDs. An error occured: {0}".format(error))
def reset_msb_db(self):
msb.init_db_engine(config.MESSYBRAINZ_ADMIN_URI)
msb.run_sql_script_without_transaction(os.path.join(MESSYBRAINZ_SQL_DIR, 'drop_db.sql'))
msb.run_sql_script_without_transaction(os.path.join(MESSYBRAINZ_SQL_DIR, 'create_db.sql'))
msb.engine.dispose()
msb.init_db_engine(config.MESSYBRAINZ_ADMIN_MSB_URI)
msb.run_sql_script_without_transaction(os.path.join(MESSYBRAINZ_SQL_DIR, 'create_extensions.sql'))
msb.engine.dispose()
msb.init_db_engine(config.MESSYBRAINZ_SQLALCHEMY_DATABASE_URI)
msb.run_sql_script(os.path.join(MESSYBRAINZ_SQL_DIR, 'create_tables.sql'))
msb.run_sql_script(os.path.join(MESSYBRAINZ_SQL_DIR, 'create_functions.sql'))
msb.run_sql_script(os.path.join(MESSYBRAINZ_SQL_DIR, 'create_indexes.sql'))
msb.run_sql_script(os.path.join(MESSYBRAINZ_SQL_DIR, 'create_primary_keys.sql'))
msb.run_sql_script(os.path.join(MESSYBRAINZ_SQL_DIR, 'create_foreign_keys.sql'))
msb.engine.dispose()
def before_request():
db.init_db_engine(app.config['SQLALCHEMY_DATABASE_URI'])
def setUp(self):
msb.init_db_engine(config.MESSYBRAINZ_ADMIN_URI)
self.reset_msb_db()
def setUp(self):
db.init_db_engine(config.POSTGRES_ADMIN_URI)
self.create_db()
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
self.init_db()
def before_request():
db.init_db_engine(app.config['SQLALCHEMY_DATABASE_URI'])
def create_app(debug=None, config_path=None):
app = CustomFlask(
import_name=__name__,
use_flask_uuid=True,
)
# Configuration
app.config.from_pyfile(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '..',
'default_config.py'))
app.config.from_pyfile(os.path.join(
os.path.dirname(os.path.realpath(__file__)), '..', 'custom_config.py'),
silent=True)
if config_path:
app.config.from_pyfile(config_path)
if debug is not None:
app.debug = debug
if app.debug and app.config['SECRET_KEY']:
app.init_debug_toolbar()
# Redis (cache)
from brainzutils import cache
try:
cache.init(
host=app.config["REDIS_HOST"],
port=app.config["REDIS_PORT"],
namespace=app.config["REDIS_NAMESPACE"],
)
except KeyError as e:
logging.error(
"Redis is not defined in config file. Error: {}".format(e))
raise
# Logging
app.init_loggers(file_config=app.config.get('LOG_FILE'),
sentry_config=app.config.get('LOG_SENTRY'))
# Extensions
from flask_uuid import FlaskUUID
FlaskUUID(app)
# Error handling
from messybrainz.webserver.errors import init_error_handlers
init_error_handlers(app)
# Template utilities
app.jinja_env.add_extension('jinja2.ext.do')
from messybrainz.webserver import utils
app.jinja_env.filters['date'] = utils.reformat_date
app.jinja_env.filters['datetime'] = utils.reformat_datetime
# Blueprints
from messybrainz.webserver.views.index import index_bp
from messybrainz.webserver.views.api import api_bp
app.register_blueprint(index_bp)
app.register_blueprint(api_bp)
db.init_db_engine(app.config['SQLALCHEMY_DATABASE_URI'])
return app
