def fetch_and_store_releases(verbose=False): """ Fetches releases from the musicbrainz database for the recording MBIDs in the recording_json table submitted while submitting a listen. It fetches only the releases for the recordings MBIDs which are not in recording_release_join table. In the end it prints to the console the total recording MBIDs it processed and the total recording MBIDs it added to the recording_release_join table. """ print("Fetching release for recording MBIDs...") if verbose: logging.basicConfig(format='%(message)s', level=logging.DEBUG) # Init databases db.init_db_engine(config.SQLALCHEMY_DATABASE_URI) musicbrainz_db.init_db_engine(config.MB_DATABASE_URI) try: logging.debug("=" * 80) num_recording_mbids_processed, num_recording_mbids_added = release.fetch_and_store_releases_for_all_recording_mbids() logging.debug("=" * 80) print("Total recording MBIDs processed: {0}.".format(num_recording_mbids_processed)) print("Total recording MBIDs added to table: {0}.".format(num_recording_mbids_added)) print("Done!") except Exception as error: print("Unable to fetch releases. An error occured: {0}".format(error)) raise
def import_musicbrainz_rows(musicbrainz_db_uri, dry_run=True, delete=False): musicbrainz_db.init_db_engine(musicbrainz_db_uri) db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI']) import_count = 0 already_imported = 0 not_found = 0 deleted = 0 if not dry_run: update_row_ids_for_exceptions() users = db_user.get_all_users() with musicbrainz_db.engine.connect() as mb_connection: with db.engine.connect() as connection: for user in users: if user.get('musicbrainz_row_id') is not None: already_imported += 1 continue name = user['musicbrainz_id'] result = mb_connection.execute(sqlalchemy.text(""" SELECT id FROM editor WHERE LOWER(name) = LOWER(:name) """), { 'name': name, }) musicbrainz_row_id = None if result.rowcount > 0: musicbrainz_row_id = result.fetchone()['id'] import_count += 1 else: print('No user with specified username in the MusicBrainz db: %s' % name) if delete: print('Deleting user %s' % name) try: delete_user(user) except NotFound: print('User %s not found in LB...' % name) not_found += 1 continue if not dry_run: connection.execute(sqlalchemy.text(""" UPDATE "user" SET musicbrainz_row_id = :musicbrainz_row_id WHERE id = :id """), { 'musicbrainz_row_id': musicbrainz_row_id, 'id': user['id'], }) print('Inserted row_id %d for user %s' % (musicbrainz_row_id, name)) print('Total number of ListenBrainz users: %d' % len(users)) print('Total number of ListenBrainz users with already imported row ids: %d' % already_imported) print('Total number of ListenBrainz users whose row ids can be imported: %d' % import_count) print('Total number of ListenBrainz users not found in MusicBrainz: %d' % not_found) print('Total number of ListenBrainz users deleted from MusicBrainz: %d' % deleted)
def fetch_and_store_artist_mbids(): """ Fetches artist MBIDs from the musicbrainz database for the recording MBIDs in the recording_json table submitted while submitting a listen. It fetches only the artist MBIDs for the recordings MBIDs which are not in recording_artist_join table. In the end it prints to the console the total recording MBIDs it processed and the total recording MBIDs it added to the recording_artist_join table. """ # Init databases db.init_db_engine(config.SQLALCHEMY_DATABASE_URI) musicbrainz_db.init_db_engine(config.MB_DATABASE_URI) try: num_recording_mbids_processed, num_recording_mbids_added = fetch_and_store_artist_mbids_for_all_recording_mbids() print("Total recording MBIDs processed: {0}.".format(num_recording_mbids_processed)) print("Total recording MBIDs added to table: {0}.".format(num_recording_mbids_added)) print("Done!") except Exception as error: print("Unable to fetch artist MBIDs. An error occured: {0}".format(error)) raise
def create_app(config_object="spambrainz.config.Config"): # create and configure the app app = Flask(__name__, instance_relative_config=False, template_folder="web/templates") app.config.from_object(config_object) db.init_app(app) if app.debug: toolbar.init_app(app) # reset_debug_db() backend_setting = app.config["BACKEND"] if backend_setting == "dummy": from .backend.dummy import DummyBackend backend = DummyBackend() else: mbdb_uri = app.config["MB_DATABASE_URI"] from brainzutils.musicbrainz_db import init_db_engine init_db_engine(mbdb_uri) if backend_setting == "dbdummy": from .backend.db_dummy import DbDummyBackend backend = DbDummyBackend() else: from .backend.celery import CeleryBackend backend = CeleryBackend() app.register_blueprint(index.bp) app.register_blueprint(create_api_bp(backend, app.config["API_TOKEN"]), url_prefix=app.config["API_PREFIX"]) return app
def import_musicbrainz_rows(musicbrainz_db_uri, dry_run=True, delete=False): musicbrainz_db.init_db_engine(musicbrainz_db_uri) db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI']) import_count = 0 already_imported = 0 not_found = 0 deleted = 0 if not dry_run: update_row_ids_for_exceptions() users = db_user.get_all_users() with musicbrainz_db.engine.connect() as mb_connection: with db.engine.connect() as connection: for user in users: if user.get('musicbrainz_row_id') is not None: already_imported += 1 continue name = user['musicbrainz_id'] result = mb_connection.execute( sqlalchemy.text(""" SELECT id FROM editor WHERE LOWER(name) = LOWER(:name) """), { 'name': name, }) musicbrainz_row_id = None if result.rowcount > 0: musicbrainz_row_id = result.fetchone()['id'] import_count += 1 else: print( 'No user with specified username in the MusicBrainz db: %s' % name) if delete: print('Deleting user %s' % name) try: delete_user(user) except NotFound: print('User %s not found in LB...' % name) not_found += 1 continue if not dry_run: connection.execute( sqlalchemy.text(""" UPDATE "user" SET musicbrainz_row_id = :musicbrainz_row_id WHERE id = :id """), { 'musicbrainz_row_id': musicbrainz_row_id, 'id': user['id'], }) print('Inserted row_id %d for user %s' % (musicbrainz_row_id, name)) print('Total number of ListenBrainz users: %d' % len(users)) print( 'Total number of ListenBrainz users with already imported row ids: %d' % already_imported) print( 'Total number of ListenBrainz users whose row ids can be imported: %d' % import_count) print('Total number of ListenBrainz users not found in MusicBrainz: %d' % not_found) print('Total number of ListenBrainz users deleted from MusicBrainz: %d' % deleted)
def create_app(debug=None, config_path=None): app = CustomFlask( import_name=__name__, use_flask_uuid=True, ) # Configuration files app.config.from_pyfile( os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', 'default_config.py')) config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', 'consul_config.py') if deploy_env: print("Checking if consul template generated config file exists: {}". format(config_file)) for _ in range(CONSUL_CONFIG_FILE_RETRY_COUNT): if not os.path.exists(config_file): sleep(1) if not os.path.exists(config_file): print( "No configuration file generated yet. Retried {} times, exiting." .format(CONSUL_CONFIG_FILE_RETRY_COUNT)) sys.exit(-1) print("Loading consul config file {}".format(config_file)) app.config.from_pyfile(config_file, silent=True) app.config.from_pyfile(os.path.join( os.path.dirname(os.path.realpath(__file__)), '..', '..', 'custom_config.py'), silent=True) if config_path: app.config.from_pyfile(config_path) if debug is not None: app.debug = debug if app.debug and app.config['SECRET_KEY']: app.init_debug_toolbar() # Error handling from critiquebrainz.frontend.error_handlers import init_error_handlers init_error_handlers(app) # Static files from critiquebrainz.frontend import static_manager static_manager.read_manifest() app.init_loggers( file_config=app.config.get("LOG_FILE"), email_config=app.config.get("LOG_EMAIL"), sentry_config=app.config.get("LOG_SENTRY"), ) # CritiqueBrainz Database from critiquebrainz import db as critiquebrainz_db critiquebrainz_db.init_db_engine(app.config.get("SQLALCHEMY_DATABASE_URI")) add_robots(app) # MusicBrainz Database from brainzutils import musicbrainz_db musicbrainz_db.init_db_engine(app.config.get("MB_DATABASE_URI")) # Redis (cache) from brainzutils import cache if "REDIS_HOST" in app.config and \ "REDIS_PORT" in app.config and \ "REDIS_NAMESPACE" in app.config: cache.init( host=app.config["REDIS_HOST"], port=app.config["REDIS_PORT"], namespace=app.config["REDIS_NAMESPACE"], ) else: logging.warning( "Redis is not defined in config file. Skipping initialization.") from critiquebrainz.frontend import babel babel.init_app(app) from critiquebrainz.frontend import login login.login_manager.init_app(app) from critiquebrainz.frontend.login.provider import MusicBrainzAuthentication login.mb_auth = MusicBrainzAuthentication( name='musicbrainz', client_id=app.config['MUSICBRAINZ_CLIENT_ID'], client_secret=app.config['MUSICBRAINZ_CLIENT_SECRET'], authorize_url="https://musicbrainz.org/oauth2/authorize", access_token_url="https://musicbrainz.org/oauth2/token", base_url="https://musicbrainz.org/", ) # APIs from critiquebrainz.frontend.external import mbspotify mbspotify.init(app.config['MBSPOTIFY_BASE_URI'], app.config['MBSPOTIFY_ACCESS_KEY']) from critiquebrainz.frontend.external import musicbrainz musicbrainz.init( app_name=app.config['MUSICBRAINZ_USERAGENT'] or "CritiqueBrainz Custom", app_version="1.0", hostname=app.config['MUSICBRAINZ_HOSTNAME'] or "musicbrainz.org", ) # Template utilities # TODO (code-master5): disabled no-member warnings just as a workaround to deal with failing tests till the # issue [https://github.com/PyCQA/pylint/issues/2563] with pylint is resolved app.jinja_env.add_extension('jinja2.ext.do') from critiquebrainz.utils import reformat_date, reformat_datetime, track_length, track_length_ms, parameterize from critiquebrainz.frontend.external.musicbrainz_db.entities import get_entity_by_id from critiquebrainz.frontend.forms.utils import get_language_name app.jinja_env.filters['date'] = reformat_date app.jinja_env.filters['datetime'] = reformat_datetime app.jinja_env.filters['track_length'] = track_length app.jinja_env.filters['track_length_ms'] = track_length_ms app.jinja_env.filters['parameterize'] = parameterize app.jinja_env.filters['entity_details'] = get_entity_by_id app.jinja_env.filters['language_name'] = get_language_name app.context_processor( lambda: dict(get_static_path=static_manager.get_static_path)) # Blueprints from critiquebrainz.frontend.views.index import frontend_bp from critiquebrainz.frontend.views.review import review_bp from critiquebrainz.frontend.views.search import search_bp from critiquebrainz.frontend.views.artist import artist_bp from critiquebrainz.frontend.views.label import label_bp from critiquebrainz.frontend.views.release_group import release_group_bp from critiquebrainz.frontend.views.release import release_bp from critiquebrainz.frontend.views.work import work_bp from critiquebrainz.frontend.views.event import event_bp from critiquebrainz.frontend.views.mapping import mapping_bp from critiquebrainz.frontend.views.user import user_bp from critiquebrainz.frontend.views.profile import profile_bp from critiquebrainz.frontend.views.place import place_bp from critiquebrainz.frontend.views.profile_apps import profile_apps_bp from critiquebrainz.frontend.views.login import login_bp from critiquebrainz.frontend.views.oauth import oauth_bp from critiquebrainz.frontend.views.reports import reports_bp from critiquebrainz.frontend.views.moderators import moderators_bp from critiquebrainz.frontend.views.log import log_bp from critiquebrainz.frontend.views.comment import comment_bp from critiquebrainz.frontend.views.rate import rate_bp from critiquebrainz.frontend.views.statistics import statistics_bp app.register_blueprint(frontend_bp) app.register_blueprint(review_bp, url_prefix='/review') app.register_blueprint(search_bp, url_prefix='/search') app.register_blueprint(artist_bp, url_prefix='/artist') app.register_blueprint(label_bp, url_prefix='/label') app.register_blueprint(release_group_bp, url_prefix='/release-group') app.register_blueprint(release_bp, url_prefix='/release') app.register_blueprint(work_bp, url_prefix='/work') app.register_blueprint(event_bp, url_prefix='/event') app.register_blueprint(place_bp, url_prefix='/place') app.register_blueprint(mapping_bp, url_prefix='/mapping') app.register_blueprint(user_bp, url_prefix='/user') app.register_blueprint(profile_bp, url_prefix='/profile') app.register_blueprint(profile_apps_bp, url_prefix='/profile/applications') app.register_blueprint(login_bp, url_prefix='/login') app.register_blueprint(oauth_bp, url_prefix='/oauth') app.register_blueprint(reports_bp, url_prefix='/reports') app.register_blueprint(log_bp, url_prefix='/log') app.register_blueprint(moderators_bp, url_prefix='/moderators') app.register_blueprint(comment_bp, url_prefix='/comments') app.register_blueprint(rate_bp, url_prefix='/rate') app.register_blueprint(statistics_bp, url_prefix='/statistics') return app
def gen_app(config_path=None, debug=None): """ Generate a Flask app for LB with all configurations done and connections established. In the Flask app returned, blueprints are not registered. """ app = CustomFlask( import_name=__name__, use_flask_uuid=True, ) load_config(app) if debug is not None: app.debug = debug # initialize Flask-DebugToolbar if the debug option is True if app.debug and app.config['SECRET_KEY']: app.init_debug_toolbar() # Logging app.init_loggers(file_config=app.config.get('LOG_FILE'), email_config=app.config.get('LOG_EMAIL'), sentry_config=app.config.get('LOG_SENTRY')) # Redis connection create_redis(app) # Influx connection create_influx(app) # RabbitMQ connection try: create_rabbitmq(app) except ConnectionError: app.logger.critical("RabbitMQ service is not up!", exc_info=True) # Database connection from listenbrainz import db db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI']) from listenbrainz.webserver.external import messybrainz messybrainz.init_db_connection( app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI']) if app.config['MB_DATABASE_URI']: from brainzutils import musicbrainz_db musicbrainz_db.init_db_engine(app.config['MB_DATABASE_URI']) # OAuth from listenbrainz.webserver.login import login_manager, provider login_manager.init_app(app) provider.init(app.config['MUSICBRAINZ_CLIENT_ID'], app.config['MUSICBRAINZ_CLIENT_SECRET']) # Error handling from listenbrainz.webserver.errors import init_error_handlers init_error_handlers(app) from listenbrainz.webserver import rate_limiter @app.after_request def after_request_callbacks(response): return rate_limiter.inject_x_rate_headers(response) # Template utilities app.jinja_env.add_extension('jinja2.ext.do') from listenbrainz.webserver import utils app.jinja_env.filters['date'] = utils.reformat_date app.jinja_env.filters['datetime'] = utils.reformat_datetime return app
def create_app(debug=None, config_path=None): app = CustomFlask( import_name=__name__, use_flask_uuid=True, ) # Configuration files app.config.from_pyfile( os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', 'default_config.py')) config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', 'consul_config.py') if deploy_env: print("Checking if consul template generated config file exists: {}". format(config_file)) for _ in range(CONSUL_CONFIG_FILE_RETRY_COUNT): if not os.path.exists(config_file): sleep(1) if not os.path.exists(config_file): print( "No configuration file generated yet. Retried {} times, exiting." .format(CONSUL_CONFIG_FILE_RETRY_COUNT)) sys.exit(-1) print("Loading consul config file {}".format(config_file)) app.config.from_pyfile(config_file, silent=True) app.config.from_pyfile(os.path.join( os.path.dirname(os.path.realpath(__file__)), '..', '..', 'custom_config.py'), silent=True) if config_path: app.config.from_pyfile(config_path) if debug is not None: app.debug = debug # Error handling from critiquebrainz.ws.errors import init_error_handlers init_error_handlers(app) app.init_loggers( file_config=app.config.get("LOG_FILE"), email_config=app.config.get("LOG_EMAIL"), sentry_config=app.config.get("LOG_SENTRY"), ) # CritiqueBrainz Database from critiquebrainz import db as critiquebrainz_db critiquebrainz_db.init_db_engine(app.config.get("SQLALCHEMY_DATABASE_URI")) # MusicBrainz Database from brainzutils import musicbrainz_db musicbrainz_db.init_db_engine(app.config.get("MB_DATABASE_URI")) # Redis (cache) from brainzutils import cache if "REDIS_HOST" in app.config and \ "REDIS_PORT" in app.config and \ "REDIS_NAMESPACE" in app.config: cache.init( host=app.config["REDIS_HOST"], port=app.config["REDIS_PORT"], namespace=app.config["REDIS_NAMESPACE"], ) else: logging.warning( "Redis is not defined in config file. Skipping initialization.") # OAuth from critiquebrainz.ws.oauth import oauth oauth.init_app(app) app.config["JSONIFY_PRETTYPRINT_REGULAR"] = False _register_blueprints(app) return app
def create_app(debug=None): """ Generate a Flask app for LB with all configurations done and connections established. In the Flask app returned, blueprints are not registered. """ app = CustomFlask( import_name=__name__, use_flask_uuid=True, ) load_config(app) if debug is not None: app.debug = debug # As early as possible, if debug is True, set the log level of our 'listenbrainz' logger to DEBUG # to prevent flask from creating a new log handler if app.debug: logger = logging.getLogger('listenbrainz') logger.setLevel(logging.DEBUG) # initialize Flask-DebugToolbar if the debug option is True if app.debug and app.config['SECRET_KEY']: app.init_debug_toolbar() sentry_config = app.config.get('LOG_SENTRY') if sentry_config: sentry.init_sentry(**sentry_config) # Initialize BU cache and metrics cache.init(host=app.config['REDIS_HOST'], port=app.config['REDIS_PORT'], namespace=app.config['REDIS_NAMESPACE']) metrics.init("listenbrainz") # Database connections from listenbrainz import db from listenbrainz.db import timescale as ts from listenbrainz import messybrainz as msb db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI']) ts.init_db_connection(app.config['SQLALCHEMY_TIMESCALE_URI']) msb.init_db_connection(app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI']) # Redis connection from listenbrainz.webserver.redis_connection import init_redis_connection init_redis_connection(app.logger) # Timescale connection from listenbrainz.webserver.timescale_connection import init_timescale_connection init_timescale_connection(app) # RabbitMQ connection from listenbrainz.webserver.rabbitmq_connection import init_rabbitmq_connection try: init_rabbitmq_connection(app) except ConnectionError: app.logger.critical("RabbitMQ service is not up!", exc_info=True) if app.config['MB_DATABASE_URI']: from brainzutils import musicbrainz_db musicbrainz_db.init_db_engine(app.config['MB_DATABASE_URI']) # OAuth from listenbrainz.webserver.login import login_manager, provider login_manager.init_app(app) provider.init(app.config['MUSICBRAINZ_CLIENT_ID'], app.config['MUSICBRAINZ_CLIENT_SECRET']) # Error handling from listenbrainz.webserver.errors import init_error_handlers init_error_handlers(app) from brainzutils.ratelimit import inject_x_rate_headers, set_user_validation_function set_user_validation_function(check_ratelimit_token_whitelist) @app.after_request def after_request_callbacks(response): return inject_x_rate_headers(response) # Template utilities app.jinja_env.add_extension('jinja2.ext.do') from listenbrainz.webserver import utils app.jinja_env.filters['date'] = utils.reformat_date app.jinja_env.filters['datetime'] = utils.reformat_datetime return app
from brainzutils import musicbrainz_db musicbrainz_db.init_db_engine("postgresql://*****:*****@musicbrainz_db:5432/musicbrainz_db") # It is already initialized while server is up i guess from sqlalchemy import text import db with db.engine.begin() as conn: lowlevel_query = text("""SELECT gid from lowlevel""") gids_in_AB = conn.execute(lowlevel_query) for recording_gid in gids_in_AB.fetchall(): MB_artist_credit_data = 0 MB_recording_data = 0 #SIMILARLY FOR OTHER TABLES # FROM MUSICBRAINZ with musicbrainz_db.engine.begin() as connection: # Make it a function artist_credit_query = text("""SELECT artist_credit.id, artist_credit.name, artist_credit.artist_count, artist_credit.ref_count, artist_credit.created FROM artist_credit INNER JOIN recording ON artist_credit.id = recording.artist_credit WHERE recording.gid= :recording""") # Correct the SYNTAX HERE result = connection.execute(artist_credit_query, {"recording" : recording[0]}) MB_artist_credit_data = result.fetchone() recording_query = text("""SELECT * from recording where gid= recording_gid""") # Correct the SYNTAX HERE result = connection.execute(recording_query) MB_recording_data = result.fetchone() # TODO: # * to write script for all tables to write to AB #### * figure out how to write those rows from a table which
def engine(): init_db_engine("postgresql://musicbrainz@musicbrainz_db/musicbrainz_db")