def fetch_and_store_releases(verbose=False):
""" Fetches releases from the musicbrainz database for the recording MBIDs
in the recording_json table submitted while submitting a listen. It fetches
only the releases for the recordings MBIDs which are not in recording_release_join
table. In the end it prints to the console the total recording MBIDs it processed
and the total recording MBIDs it added to the recording_release_join table.
"""
print("Fetching release for recording MBIDs...")
if verbose:
logging.basicConfig(format='%(message)s', level=logging.DEBUG)
# Init databases
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
musicbrainz_db.init_db_engine(config.MB_DATABASE_URI)
try:
logging.debug("=" * 80)
num_recording_mbids_processed, num_recording_mbids_added = release.fetch_and_store_releases_for_all_recording_mbids()
logging.debug("=" * 80)
print("Total recording MBIDs processed: {0}.".format(num_recording_mbids_processed))
print("Total recording MBIDs added to table: {0}.".format(num_recording_mbids_added))
print("Done!")
except Exception as error:
print("Unable to fetch releases. An error occured: {0}".format(error))
raise
def import_musicbrainz_rows(musicbrainz_db_uri, dry_run=True, delete=False):
musicbrainz_db.init_db_engine(musicbrainz_db_uri)
db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI'])
import_count = 0
already_imported = 0
not_found = 0
deleted = 0
if not dry_run:
update_row_ids_for_exceptions()
users = db_user.get_all_users()
with musicbrainz_db.engine.connect() as mb_connection:
with db.engine.connect() as connection:
for user in users:
if user.get('musicbrainz_row_id') is not None:
already_imported += 1
continue
name = user['musicbrainz_id']
result = mb_connection.execute(sqlalchemy.text("""
SELECT id
FROM editor
WHERE LOWER(name) = LOWER(:name)
"""), {
'name': name,
})
musicbrainz_row_id = None
if result.rowcount > 0:
musicbrainz_row_id = result.fetchone()['id']
import_count += 1
else:
print('No user with specified username in the MusicBrainz db: %s' % name)
if delete:
print('Deleting user %s' % name)
try:
delete_user(user)
except NotFound:
print('User %s not found in LB...' % name)
not_found += 1
continue
if not dry_run:
connection.execute(sqlalchemy.text("""
UPDATE "user"
SET musicbrainz_row_id = :musicbrainz_row_id
WHERE id = :id
"""), {
'musicbrainz_row_id': musicbrainz_row_id,
'id': user['id'],
})
print('Inserted row_id %d for user %s' % (musicbrainz_row_id, name))
print('Total number of ListenBrainz users: %d' % len(users))
print('Total number of ListenBrainz users with already imported row ids: %d' % already_imported)
print('Total number of ListenBrainz users whose row ids can be imported: %d' % import_count)
print('Total number of ListenBrainz users not found in MusicBrainz: %d' % not_found)
print('Total number of ListenBrainz users deleted from MusicBrainz: %d' % deleted)
def fetch_and_store_artist_mbids():
""" Fetches artist MBIDs from the musicbrainz database for the recording MBIDs
in the recording_json table submitted while submitting a listen. It fetches
only the artist MBIDs for the recordings MBIDs which are not in recording_artist_join
table. In the end it prints to the console the total recording MBIDs it processed
and the total recording MBIDs it added to the recording_artist_join table.
"""
# Init databases
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
musicbrainz_db.init_db_engine(config.MB_DATABASE_URI)
try:
num_recording_mbids_processed, num_recording_mbids_added = fetch_and_store_artist_mbids_for_all_recording_mbids()
print("Total recording MBIDs processed: {0}.".format(num_recording_mbids_processed))
print("Total recording MBIDs added to table: {0}.".format(num_recording_mbids_added))
print("Done!")
except Exception as error:
print("Unable to fetch artist MBIDs. An error occured: {0}".format(error))
raise
def create_app(config_object="spambrainz.config.Config"):
# create and configure the app
app = Flask(__name__,
instance_relative_config=False,
template_folder="web/templates")
app.config.from_object(config_object)
db.init_app(app)
if app.debug:
toolbar.init_app(app)
# reset_debug_db()
backend_setting = app.config["BACKEND"]
if backend_setting == "dummy":
from .backend.dummy import DummyBackend
backend = DummyBackend()
else:
mbdb_uri = app.config["MB_DATABASE_URI"]
from brainzutils.musicbrainz_db import init_db_engine
init_db_engine(mbdb_uri)
if backend_setting == "dbdummy":
from .backend.db_dummy import DbDummyBackend
backend = DbDummyBackend()
else:
from .backend.celery import CeleryBackend
backend = CeleryBackend()
app.register_blueprint(index.bp)
app.register_blueprint(create_api_bp(backend, app.config["API_TOKEN"]),
url_prefix=app.config["API_PREFIX"])
return app
def import_musicbrainz_rows(musicbrainz_db_uri, dry_run=True, delete=False):
musicbrainz_db.init_db_engine(musicbrainz_db_uri)
db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI'])
import_count = 0
already_imported = 0
not_found = 0
deleted = 0
if not dry_run:
update_row_ids_for_exceptions()
users = db_user.get_all_users()
with musicbrainz_db.engine.connect() as mb_connection:
with db.engine.connect() as connection:
for user in users:
if user.get('musicbrainz_row_id') is not None:
already_imported += 1
continue
name = user['musicbrainz_id']
result = mb_connection.execute(
sqlalchemy.text("""
SELECT id
FROM editor
WHERE LOWER(name) = LOWER(:name)
"""), {
'name': name,
})
musicbrainz_row_id = None
if result.rowcount > 0:
musicbrainz_row_id = result.fetchone()['id']
import_count += 1
else:
print(
'No user with specified username in the MusicBrainz db: %s'
% name)
if delete:
print('Deleting user %s' % name)
try:
delete_user(user)
except NotFound:
print('User %s not found in LB...' % name)
not_found += 1
continue
if not dry_run:
connection.execute(
sqlalchemy.text("""
UPDATE "user"
SET musicbrainz_row_id = :musicbrainz_row_id
WHERE id = :id
"""), {
'musicbrainz_row_id': musicbrainz_row_id,
'id': user['id'],
})
print('Inserted row_id %d for user %s' %
(musicbrainz_row_id, name))
print('Total number of ListenBrainz users: %d' % len(users))
print(
'Total number of ListenBrainz users with already imported row ids: %d'
% already_imported)
print(
'Total number of ListenBrainz users whose row ids can be imported: %d'
% import_count)
print('Total number of ListenBrainz users not found in MusicBrainz: %d' %
not_found)
print('Total number of ListenBrainz users deleted from MusicBrainz: %d' %
deleted)
def create_app(debug=None, config_path=None):
app = CustomFlask(
import_name=__name__,
use_flask_uuid=True,
)
# Configuration files
app.config.from_pyfile(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..',
'default_config.py'))
config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'..', '..', 'consul_config.py')
if deploy_env:
print("Checking if consul template generated config file exists: {}".
format(config_file))
for _ in range(CONSUL_CONFIG_FILE_RETRY_COUNT):
if not os.path.exists(config_file):
sleep(1)
if not os.path.exists(config_file):
print(
"No configuration file generated yet. Retried {} times, exiting."
.format(CONSUL_CONFIG_FILE_RETRY_COUNT))
sys.exit(-1)
print("Loading consul config file {}".format(config_file))
app.config.from_pyfile(config_file, silent=True)
app.config.from_pyfile(os.path.join(
os.path.dirname(os.path.realpath(__file__)), '..', '..',
'custom_config.py'),
silent=True)
if config_path:
app.config.from_pyfile(config_path)
if debug is not None:
app.debug = debug
if app.debug and app.config['SECRET_KEY']:
app.init_debug_toolbar()
# Error handling
from critiquebrainz.frontend.error_handlers import init_error_handlers
init_error_handlers(app)
# Static files
from critiquebrainz.frontend import static_manager
static_manager.read_manifest()
app.init_loggers(
file_config=app.config.get("LOG_FILE"),
email_config=app.config.get("LOG_EMAIL"),
sentry_config=app.config.get("LOG_SENTRY"),
)
# CritiqueBrainz Database
from critiquebrainz import db as critiquebrainz_db
critiquebrainz_db.init_db_engine(app.config.get("SQLALCHEMY_DATABASE_URI"))
add_robots(app)
# MusicBrainz Database
from brainzutils import musicbrainz_db
musicbrainz_db.init_db_engine(app.config.get("MB_DATABASE_URI"))
# Redis (cache)
from brainzutils import cache
if "REDIS_HOST" in app.config and \
"REDIS_PORT" in app.config and \
"REDIS_NAMESPACE" in app.config:
cache.init(
host=app.config["REDIS_HOST"],
port=app.config["REDIS_PORT"],
namespace=app.config["REDIS_NAMESPACE"],
)
else:
logging.warning(
"Redis is not defined in config file. Skipping initialization.")
from critiquebrainz.frontend import babel
babel.init_app(app)
from critiquebrainz.frontend import login
login.login_manager.init_app(app)
from critiquebrainz.frontend.login.provider import MusicBrainzAuthentication
login.mb_auth = MusicBrainzAuthentication(
name='musicbrainz',
client_id=app.config['MUSICBRAINZ_CLIENT_ID'],
client_secret=app.config['MUSICBRAINZ_CLIENT_SECRET'],
authorize_url="https://musicbrainz.org/oauth2/authorize",
access_token_url="https://musicbrainz.org/oauth2/token",
base_url="https://musicbrainz.org/",
)
# APIs
from critiquebrainz.frontend.external import mbspotify
mbspotify.init(app.config['MBSPOTIFY_BASE_URI'],
app.config['MBSPOTIFY_ACCESS_KEY'])
from critiquebrainz.frontend.external import musicbrainz
musicbrainz.init(
app_name=app.config['MUSICBRAINZ_USERAGENT']
or "CritiqueBrainz Custom",
app_version="1.0",
hostname=app.config['MUSICBRAINZ_HOSTNAME'] or "musicbrainz.org",
)
# Template utilities
# TODO (code-master5): disabled no-member warnings just as a workaround to deal with failing tests till the
# issue [https://github.com/PyCQA/pylint/issues/2563] with pylint is resolved
app.jinja_env.add_extension('jinja2.ext.do')
from critiquebrainz.utils import reformat_date, reformat_datetime, track_length, track_length_ms, parameterize
from critiquebrainz.frontend.external.musicbrainz_db.entities import get_entity_by_id
from critiquebrainz.frontend.forms.utils import get_language_name
app.jinja_env.filters['date'] = reformat_date
app.jinja_env.filters['datetime'] = reformat_datetime
app.jinja_env.filters['track_length'] = track_length
app.jinja_env.filters['track_length_ms'] = track_length_ms
app.jinja_env.filters['parameterize'] = parameterize
app.jinja_env.filters['entity_details'] = get_entity_by_id
app.jinja_env.filters['language_name'] = get_language_name
app.context_processor(
lambda: dict(get_static_path=static_manager.get_static_path))
# Blueprints
from critiquebrainz.frontend.views.index import frontend_bp
from critiquebrainz.frontend.views.review import review_bp
from critiquebrainz.frontend.views.search import search_bp
from critiquebrainz.frontend.views.artist import artist_bp
from critiquebrainz.frontend.views.label import label_bp
from critiquebrainz.frontend.views.release_group import release_group_bp
from critiquebrainz.frontend.views.release import release_bp
from critiquebrainz.frontend.views.work import work_bp
from critiquebrainz.frontend.views.event import event_bp
from critiquebrainz.frontend.views.mapping import mapping_bp
from critiquebrainz.frontend.views.user import user_bp
from critiquebrainz.frontend.views.profile import profile_bp
from critiquebrainz.frontend.views.place import place_bp
from critiquebrainz.frontend.views.profile_apps import profile_apps_bp
from critiquebrainz.frontend.views.login import login_bp
from critiquebrainz.frontend.views.oauth import oauth_bp
from critiquebrainz.frontend.views.reports import reports_bp
from critiquebrainz.frontend.views.moderators import moderators_bp
from critiquebrainz.frontend.views.log import log_bp
from critiquebrainz.frontend.views.comment import comment_bp
from critiquebrainz.frontend.views.rate import rate_bp
from critiquebrainz.frontend.views.statistics import statistics_bp
app.register_blueprint(frontend_bp)
app.register_blueprint(review_bp, url_prefix='/review')
app.register_blueprint(search_bp, url_prefix='/search')
app.register_blueprint(artist_bp, url_prefix='/artist')
app.register_blueprint(label_bp, url_prefix='/label')
app.register_blueprint(release_group_bp, url_prefix='/release-group')
app.register_blueprint(release_bp, url_prefix='/release')
app.register_blueprint(work_bp, url_prefix='/work')
app.register_blueprint(event_bp, url_prefix='/event')
app.register_blueprint(place_bp, url_prefix='/place')
app.register_blueprint(mapping_bp, url_prefix='/mapping')
app.register_blueprint(user_bp, url_prefix='/user')
app.register_blueprint(profile_bp, url_prefix='/profile')
app.register_blueprint(profile_apps_bp, url_prefix='/profile/applications')
app.register_blueprint(login_bp, url_prefix='/login')
app.register_blueprint(oauth_bp, url_prefix='/oauth')
app.register_blueprint(reports_bp, url_prefix='/reports')
app.register_blueprint(log_bp, url_prefix='/log')
app.register_blueprint(moderators_bp, url_prefix='/moderators')
app.register_blueprint(comment_bp, url_prefix='/comments')
app.register_blueprint(rate_bp, url_prefix='/rate')
app.register_blueprint(statistics_bp, url_prefix='/statistics')
return app
def gen_app(config_path=None, debug=None):
""" Generate a Flask app for LB with all configurations done and connections established.
In the Flask app returned, blueprints are not registered.
"""
app = CustomFlask(
import_name=__name__,
use_flask_uuid=True,
)
load_config(app)
if debug is not None:
app.debug = debug
# initialize Flask-DebugToolbar if the debug option is True
if app.debug and app.config['SECRET_KEY']:
app.init_debug_toolbar()
# Logging
app.init_loggers(file_config=app.config.get('LOG_FILE'),
email_config=app.config.get('LOG_EMAIL'),
sentry_config=app.config.get('LOG_SENTRY'))
# Redis connection
create_redis(app)
# Influx connection
create_influx(app)
# RabbitMQ connection
try:
create_rabbitmq(app)
except ConnectionError:
app.logger.critical("RabbitMQ service is not up!", exc_info=True)
# Database connection
from listenbrainz import db
db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI'])
from listenbrainz.webserver.external import messybrainz
messybrainz.init_db_connection(
app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI'])
if app.config['MB_DATABASE_URI']:
from brainzutils import musicbrainz_db
musicbrainz_db.init_db_engine(app.config['MB_DATABASE_URI'])
# OAuth
from listenbrainz.webserver.login import login_manager, provider
login_manager.init_app(app)
provider.init(app.config['MUSICBRAINZ_CLIENT_ID'],
app.config['MUSICBRAINZ_CLIENT_SECRET'])
# Error handling
from listenbrainz.webserver.errors import init_error_handlers
init_error_handlers(app)
from listenbrainz.webserver import rate_limiter
@app.after_request
def after_request_callbacks(response):
return rate_limiter.inject_x_rate_headers(response)
# Template utilities
app.jinja_env.add_extension('jinja2.ext.do')
from listenbrainz.webserver import utils
app.jinja_env.filters['date'] = utils.reformat_date
app.jinja_env.filters['datetime'] = utils.reformat_datetime
return app
def create_app(debug=None, config_path=None):
app = CustomFlask(
import_name=__name__,
use_flask_uuid=True,
)
# Configuration files
app.config.from_pyfile(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..',
'default_config.py'))
config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'..', '..', 'consul_config.py')
if deploy_env:
print("Checking if consul template generated config file exists: {}".
format(config_file))
for _ in range(CONSUL_CONFIG_FILE_RETRY_COUNT):
if not os.path.exists(config_file):
sleep(1)
if not os.path.exists(config_file):
print(
"No configuration file generated yet. Retried {} times, exiting."
.format(CONSUL_CONFIG_FILE_RETRY_COUNT))
sys.exit(-1)
print("Loading consul config file {}".format(config_file))
app.config.from_pyfile(config_file, silent=True)
app.config.from_pyfile(os.path.join(
os.path.dirname(os.path.realpath(__file__)), '..', '..',
'custom_config.py'),
silent=True)
if config_path:
app.config.from_pyfile(config_path)
if debug is not None:
app.debug = debug
# Error handling
from critiquebrainz.ws.errors import init_error_handlers
init_error_handlers(app)
app.init_loggers(
file_config=app.config.get("LOG_FILE"),
email_config=app.config.get("LOG_EMAIL"),
sentry_config=app.config.get("LOG_SENTRY"),
)
# CritiqueBrainz Database
from critiquebrainz import db as critiquebrainz_db
critiquebrainz_db.init_db_engine(app.config.get("SQLALCHEMY_DATABASE_URI"))
# MusicBrainz Database
from brainzutils import musicbrainz_db
musicbrainz_db.init_db_engine(app.config.get("MB_DATABASE_URI"))
# Redis (cache)
from brainzutils import cache
if "REDIS_HOST" in app.config and \
"REDIS_PORT" in app.config and \
"REDIS_NAMESPACE" in app.config:
cache.init(
host=app.config["REDIS_HOST"],
port=app.config["REDIS_PORT"],
namespace=app.config["REDIS_NAMESPACE"],
)
else:
logging.warning(
"Redis is not defined in config file. Skipping initialization.")
# OAuth
from critiquebrainz.ws.oauth import oauth
oauth.init_app(app)
app.config["JSONIFY_PRETTYPRINT_REGULAR"] = False
_register_blueprints(app)
return app
def create_app(debug=None):
""" Generate a Flask app for LB with all configurations done and connections established.
In the Flask app returned, blueprints are not registered.
"""
app = CustomFlask(
import_name=__name__,
use_flask_uuid=True,
)
load_config(app)
if debug is not None:
app.debug = debug
# As early as possible, if debug is True, set the log level of our 'listenbrainz' logger to DEBUG
# to prevent flask from creating a new log handler
if app.debug:
logger = logging.getLogger('listenbrainz')
logger.setLevel(logging.DEBUG)
# initialize Flask-DebugToolbar if the debug option is True
if app.debug and app.config['SECRET_KEY']:
app.init_debug_toolbar()
sentry_config = app.config.get('LOG_SENTRY')
if sentry_config:
sentry.init_sentry(**sentry_config)
# Initialize BU cache and metrics
cache.init(host=app.config['REDIS_HOST'],
port=app.config['REDIS_PORT'],
namespace=app.config['REDIS_NAMESPACE'])
metrics.init("listenbrainz")
# Database connections
from listenbrainz import db
from listenbrainz.db import timescale as ts
from listenbrainz import messybrainz as msb
db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI'])
ts.init_db_connection(app.config['SQLALCHEMY_TIMESCALE_URI'])
msb.init_db_connection(app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI'])
# Redis connection
from listenbrainz.webserver.redis_connection import init_redis_connection
init_redis_connection(app.logger)
# Timescale connection
from listenbrainz.webserver.timescale_connection import init_timescale_connection
init_timescale_connection(app)
# RabbitMQ connection
from listenbrainz.webserver.rabbitmq_connection import init_rabbitmq_connection
try:
init_rabbitmq_connection(app)
except ConnectionError:
app.logger.critical("RabbitMQ service is not up!", exc_info=True)
if app.config['MB_DATABASE_URI']:
from brainzutils import musicbrainz_db
musicbrainz_db.init_db_engine(app.config['MB_DATABASE_URI'])
# OAuth
from listenbrainz.webserver.login import login_manager, provider
login_manager.init_app(app)
provider.init(app.config['MUSICBRAINZ_CLIENT_ID'],
app.config['MUSICBRAINZ_CLIENT_SECRET'])
# Error handling
from listenbrainz.webserver.errors import init_error_handlers
init_error_handlers(app)
from brainzutils.ratelimit import inject_x_rate_headers, set_user_validation_function
set_user_validation_function(check_ratelimit_token_whitelist)
@app.after_request
def after_request_callbacks(response):
return inject_x_rate_headers(response)
# Template utilities
app.jinja_env.add_extension('jinja2.ext.do')
from listenbrainz.webserver import utils
app.jinja_env.filters['date'] = utils.reformat_date
app.jinja_env.filters['datetime'] = utils.reformat_datetime
return app
from brainzutils import musicbrainz_db
musicbrainz_db.init_db_engine("postgresql://*****:*****@musicbrainz_db:5432/musicbrainz_db") # It is already initialized while server is up i guess
from sqlalchemy import text
import db
with db.engine.begin() as conn:
lowlevel_query = text("""SELECT gid from lowlevel""")
gids_in_AB = conn.execute(lowlevel_query)
for recording_gid in gids_in_AB.fetchall():
MB_artist_credit_data = 0
MB_recording_data = 0
#SIMILARLY FOR OTHER TABLES
# FROM MUSICBRAINZ
with musicbrainz_db.engine.begin() as connection: # Make it a function
artist_credit_query = text("""SELECT artist_credit.id, artist_credit.name, artist_credit.artist_count,
artist_credit.ref_count, artist_credit.created
FROM artist_credit
INNER JOIN recording
ON artist_credit.id = recording.artist_credit
WHERE recording.gid= :recording""") # Correct the SYNTAX HERE
result = connection.execute(artist_credit_query, {"recording" : recording[0]})
MB_artist_credit_data = result.fetchone()
recording_query = text("""SELECT * from recording where gid= recording_gid""") # Correct the SYNTAX HERE
result = connection.execute(recording_query)
MB_recording_data = result.fetchone()
# TODO:
# * to write script for all tables to write to AB
#### * figure out how to write those rows from a table which
def engine():
init_db_engine("postgresql://musicbrainz@musicbrainz_db/musicbrainz_db")
