def create(location, threads): """ Create a ListenBrainz data dump which includes a private dump, a statistics dump and a dump of the actual listens from InfluxDB Args: location (str): path to the directory where the dump should be made threads (int): the number of threads to be used while compression """ db.init_db_connection(config.SQLALCHEMY_DATABASE_URI) ls = init_influx_connection(log, { 'REDIS_HOST': config.REDIS_HOST, 'REDIS_PORT': config.REDIS_PORT, 'REDIS_NAMESPACE': config.REDIS_NAMESPACE, 'INFLUX_HOST': config.INFLUX_HOST, 'INFLUX_PORT': config.INFLUX_PORT, 'INFLUX_DB_NAME': config.INFLUX_DB_NAME, }) time_now = datetime.today() dump_path = os.path.join(location, 'listenbrainz-dump-{time}'.format(time=time_now.strftime('%Y%m%d-%H%M%S'))) create_path(dump_path) db_dump.dump_postgres_db(dump_path, time_now, threads) ls.dump_listens(dump_path, time_now, threads) try: write_hashes(dump_path) except IOError as e: log.error('Unable to create hash files! Error: %s', str(e)) return log.info('Dumps created and hashes written at %s' % dump_path)
def init_test_db(force=False): """Same as `init_db` command, but creates a database that will be used to run tests and doesn't import data (no need to do that). the `PG_CONNECT_TEST` variable must be defined in the config file. """ db.init_db_connection(config.POSTGRES_ADMIN_URI) if force: res = db.run_sql_script_without_transaction( os.path.join(ADMIN_SQL_DIR, 'drop_test_db.sql')) if not res: raise Exception( 'Failed to drop existing database and user! Exit code: %i' % res) print('Creating user and a database for testing...') res = db.run_sql_script_without_transaction( os.path.join(ADMIN_SQL_DIR, 'create_test_db.sql')) if not res: raise Exception( 'Failed to create test user and database! Exit code: %i' % res) res = db.run_sql_script_without_transaction( os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql')) # Don't raise an exception if the extension already exists db.engine.dispose() print("Done!")
def import_musicbrainz_rows(musicbrainz_db_uri, dry_run=True, delete=False): musicbrainz_db.init_db_engine(musicbrainz_db_uri) db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI']) import_count = 0 already_imported = 0 not_found = 0 deleted = 0 if not dry_run: update_row_ids_for_exceptions() users = db_user.get_all_users() with musicbrainz_db.engine.connect() as mb_connection: with db.engine.connect() as connection: for user in users: if user.get('musicbrainz_row_id') is not None: already_imported += 1 continue name = user['musicbrainz_id'] result = mb_connection.execute(sqlalchemy.text(""" SELECT id FROM editor WHERE LOWER(name) = LOWER(:name) """), { 'name': name, }) musicbrainz_row_id = None if result.rowcount > 0: musicbrainz_row_id = result.fetchone()['id'] import_count += 1 else: print('No user with specified username in the MusicBrainz db: %s' % name) if delete: print('Deleting user %s' % name) try: delete_user(user) except NotFound: print('User %s not found in LB...' % name) not_found += 1 continue if not dry_run: connection.execute(sqlalchemy.text(""" UPDATE "user" SET musicbrainz_row_id = :musicbrainz_row_id WHERE id = :id """), { 'musicbrainz_row_id': musicbrainz_row_id, 'id': user['id'], }) print('Inserted row_id %d for user %s' % (musicbrainz_row_id, name)) print('Total number of ListenBrainz users: %d' % len(users)) print('Total number of ListenBrainz users with already imported row ids: %d' % already_imported) print('Total number of ListenBrainz users whose row ids can be imported: %d' % import_count) print('Total number of ListenBrainz users not found in MusicBrainz: %d' % not_found) print('Total number of ListenBrainz users deleted from MusicBrainz: %d' % deleted)
def init_msb_db(force, create_db): """Initializes database. This process involves several steps: 1. Table structure is created. 2. Primary keys and foreign keys are created. 3. Indexes are created. """ from listenbrainz import config db.init_db_connection(config.POSTGRES_ADMIN_URI) if force: res = db.run_sql_script_without_transaction( os.path.join(MSB_ADMIN_SQL_DIR, 'drop_db.sql')) if not res: raise Exception( 'Failed to drop existing database and user! Exit code: %s' % res) if create_db: print('Creating user and a database...') res = db.run_sql_script_without_transaction( os.path.join(MSB_ADMIN_SQL_DIR, 'create_db.sql')) if not res: raise Exception( 'Failed to create new database and user! Exit code: %s' % res) print('Creating database extensions...') res = db.run_sql_script_without_transaction( os.path.join(MSB_ADMIN_SQL_DIR, 'create_extensions.sql')) # Don't raise an exception if the extension already exists db.engine.dispose() # print('Creating schema...') # exit_code = run_psql_script('create_schema.sql') # if exit_code != 0: # raise Exception('Failed to create database schema! Exit code: %i' % exit_code) db.init_db_connection(config.MESSYBRAINZ_SQLALCHEMY_DATABASE_URI) print('Creating tables...') db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_tables.sql')) print('Creating primary and foreign keys...') db.run_sql_script( os.path.join(MSB_ADMIN_SQL_DIR, 'create_primary_keys.sql')) db.run_sql_script( os.path.join(MSB_ADMIN_SQL_DIR, 'create_foreign_keys.sql')) print('Creating functions...') db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_functions.sql')) print('Creating indexes...') db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_indexes.sql')) print("Done!")
def populate_queue(force): """ Populate the ListenBrainz stats calculation queue with entities which need their stats calculated. """ logger.info('Connecting to database...') db.init_db_connection(config.SQLALCHEMY_DATABASE_URI) logger.info('Connected!') logger.info('Pushing entities whose stats should be calculated into the queue...') push_entities_to_queue(force=force) logger.info('Pushed all relevant entities, stats_calculator should calculate stats soon!')
def init_db(force, create_db): """Initializes database. This process involves several steps: 1. Table structure is created. 2. Primary keys and foreign keys are created. 3. Indexes are created. """ from listenbrainz import config db.init_db_connection(config.POSTGRES_ADMIN_URI) if force: res = db.run_sql_script_without_transaction( os.path.join(ADMIN_SQL_DIR, 'drop_db.sql')) if not res: raise Exception( 'Failed to drop existing database and user! Exit code: %i' % res) if create_db or force: print('PG: Creating user and a database...') res = db.run_sql_script_without_transaction( os.path.join(ADMIN_SQL_DIR, 'create_db.sql')) if not res: raise Exception( 'Failed to create new database and user! Exit code: %i' % res) db.init_db_connection(config.POSTGRES_ADMIN_LB_URI) print('PG: Creating database extensions...') res = db.run_sql_script_without_transaction( os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql')) # Don't raise an exception if the extension already exists application = webserver.create_app() with application.app_context(): print('PG: Creating schema...') db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_schema.sql')) print('PG: Creating Types...') db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_types.sql')) print('PG: Creating tables...') db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql')) print('PG: Creating primary and foreign keys...') db.run_sql_script( os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql')) db.run_sql_script( os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql')) print('PG: Creating indexes...') db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql')) print("Done!")
def create_app(): app = Flask(__name__) # Configuration from listenbrainz import config app.config.from_object(config) # Logging from listenbrainz.webserver.loggers import init_loggers init_loggers(app) # Redis connection create_redis(app) # Influx connection create_influx(app) # RabbitMQ connection create_rabbitmq(app) # Database connection from listenbrainz import db db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI']) from listenbrainz.webserver.external import messybrainz messybrainz.init_db_connection( app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI']) # OAuth from listenbrainz.webserver.login import login_manager, provider login_manager.init_app(app) provider.init(app.config['MUSICBRAINZ_CLIENT_ID'], app.config['MUSICBRAINZ_CLIENT_SECRET']) # Error handling from listenbrainz.webserver.errors import init_error_handlers init_error_handlers(app) from listenbrainz.webserver import rate_limiter @app.after_request def after_request_callbacks(response): return rate_limiter.inject_x_rate_headers(response) # Template utilities app.jinja_env.add_extension('jinja2.ext.do') from listenbrainz.webserver import utils app.jinja_env.filters['date'] = utils.reformat_date app.jinja_env.filters['datetime'] = utils.reformat_datetime _register_blueprints(app) return app
def import_db(location, threads=None): """ Import a ListenBrainz PostgreSQL dump into the PostgreSQL database. Note: This method tries to import the private dump first, followed by the statistics dump. However, in absence of a private dump, it imports sanitized versions of the user table in the statistics dump in order to satisfy foreign key constraints. Args: location (str): path to the directory which contains the private and the stats dump threads (int): the number of threads to use during decompression, defaults to 1 """ db.init_db_connection(config.SQLALCHEMY_DATABASE_URI) db_dump.import_postgres_dump(location, threads)
def populate_queue(force): """ Populate the ListenBrainz stats calculation queue with entities which need their stats calculated. """ logger.info('Connecting to database...') db.init_db_connection(config.SQLALCHEMY_DATABASE_URI) logger.info('Connected!') logger.info( 'Pushing entities whose stats should be calculated into the queue...') push_entities_to_queue(force=force) logger.info( 'Pushed all relevant entities, stats_calculator should calculate stats soon!' )
def import_dump(private_archive, public_archive, listen_archive, threads): """ Import a ListenBrainz dump into the database. Note: This method tries to import the private db dump first, followed by the public db dump. However, in absence of a private dump, it imports sanitized versions of the user table in the public dump in order to satisfy foreign key constraints. Then it imports the listen dump. Args: private_archive (str): the path to the ListenBrainz private dump to be imported public_archive (str): the path to the ListenBrainz public dump to be imported listen_archive (str): the path to the ListenBrainz listen dump archive to be imported threads (int): the number of threads to use during decompression, defaults to 1 """ if not private_archive and not public_archive and not listen_archive: print('You need to enter a path to the archive(s) to import!') sys.exit(1) db.init_db_connection(config.SQLALCHEMY_DATABASE_URI) db_dump.import_postgres_dump(private_archive, public_archive, threads) ls = init_influx_connection( log, { 'REDIS_HOST': config.REDIS_HOST, 'REDIS_PORT': config.REDIS_PORT, 'REDIS_NAMESPACE': config.REDIS_NAMESPACE, 'INFLUX_HOST': config.INFLUX_HOST, 'INFLUX_PORT': config.INFLUX_PORT, 'INFLUX_DB_NAME': config.INFLUX_DB_NAME, }) try: ls.import_listens_dump(listen_archive, threads) except IOError as e: log.error('IOError while trying to import data into Influx: %s', str(e)) raise except InfluxDBClientError as e: log.error('Error while sending data to Influx: %s', str(e)) raise except InfluxDBServerError as e: log.error('InfluxDB Server Error while importing data: %s', str(e)) raise except Exception as e: log.error('Unexpected error while importing data: %s', str(e)) raise
def init_msb_db(force, create_db): """Initializes database. This process involves several steps: 1. Table structure is created. 2. Primary keys and foreign keys are created. 3. Indexes are created. """ db.init_db_connection(config.POSTGRES_ADMIN_URI) if force: res = db.run_sql_script_without_transaction(os.path.join(MSB_ADMIN_SQL_DIR, 'drop_db.sql')) if not res: raise Exception('Failed to drop existing database and user! Exit code: %s' % res) if create_db: print('Creating user and a database...') res = db.run_sql_script_without_transaction(os.path.join(MSB_ADMIN_SQL_DIR, 'create_db.sql')) if not res: raise Exception('Failed to create new database and user! Exit code: %s' % res) print('Creating database extensions...') res = db.run_sql_script_without_transaction(os.path.join(MSB_ADMIN_SQL_DIR, 'create_extensions.sql')) # Don't raise an exception if the extension already exists db.engine.dispose() # print('Creating schema...') # exit_code = run_psql_script('create_schema.sql') # if exit_code != 0: # raise Exception('Failed to create database schema! Exit code: %i' % exit_code) db.init_db_connection(config.MESSYBRAINZ_SQLALCHEMY_DATABASE_URI) print('Creating tables...') db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_tables.sql')) print('Creating primary and foreign keys...') db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_primary_keys.sql')) db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_foreign_keys.sql')) print('Creating indexes...') db.run_sql_script(os.path.join(MSB_ADMIN_SQL_DIR, 'create_indexes.sql')) print("Done!")
def start(self): """ Starts the stats calculator. This should run perpetually, monitor the stats rabbitmq queue and calculate stats for entities that it receives from the queue. """ # if no bigquery support, sleep if not self.app.config['WRITE_TO_BIGQUERY']: while True: time.sleep(10000) self.log.info('Connecting to Google BigQuery...') stats.init_bigquery_connection() self.log.info('Connected!') self.log.info('Connecting to database...') db.init_db_connection(self.app.config['SQLALCHEMY_DATABASE_URI']) self.log.info('Connected!') self.log.info('Connecting to redis...') self.redis = utils.connect_to_redis(host=self.app.config['REDIS_HOST'], port=self.app.config['REDIS_PORT'], log=self.log.error) self.log.info('Connected!') while True: self.init_rabbitmq_connection() self.incoming_ch = utils.create_channel_to_consume( connection=self.connection, exchange=self.app.config['STATS_EXCHANGE'], queue=self.app.config['STATS_QUEUE'], callback_function=self.callback, ) self.log.info('Stats calculator started!') try: self.incoming_ch.start_consuming() except pika.exceptions.ConnectionClosed: self.log.info("Connection to rabbitmq closed. Re-opening.") self.connection = None continue self.connection.close()
def start(self): """ Starts the job runner. This should run perpetually, monitor the bigquery jobs rabbitmq queue and perform tasks for entries in the queue. """ with self.app.app_context(): # if no bigquery support, sleep if not current_app.config['WRITE_TO_BIGQUERY']: while True: time.sleep(10000) current_app.logger.info('Connecting to Google BigQuery...') self.bigquery = bigquery.create_bigquery_object() current_app.logger.info('Connected!') current_app.logger.info('Connecting to database...') db.init_db_connection(current_app.config['SQLALCHEMY_DATABASE_URI']) current_app.logger.info('Connected!') current_app.logger.info('Connecting to redis...') self.redis = utils.connect_to_redis(host=current_app.config['REDIS_HOST'], port=current_app.config['REDIS_PORT'], log=current_app.logger.error) current_app.logger.info('Connected!') while True: self.init_rabbitmq_connection() self.incoming_ch = utils.create_channel_to_consume( connection=self.connection, exchange=current_app.config['BIGQUERY_EXCHANGE'], queue=current_app.config['BIGQUERY_QUEUE'], callback_function=self.callback, ) current_app.logger.info('Stats calculator started!') try: self.incoming_ch.start_consuming() except pika.exceptions.ConnectionClosed: current_app.logger.warning("Connection to rabbitmq closed. Re-opening.") self.connection = None continue self.connection.close()
def init_db(force, create_db): """Initializes database. This process involves several steps: 1. Table structure is created. 2. Primary keys and foreign keys are created. 3. Indexes are created. """ db.init_db_connection(config.POSTGRES_ADMIN_URI) if force: res = db.run_sql_script_without_transaction(os.path.join(ADMIN_SQL_DIR, 'drop_db.sql')) if not res: raise Exception('Failed to drop existing database and user! Exit code: %i' % res) if create_db: print('Creating user and a database...') res = db.run_sql_script_without_transaction(os.path.join(ADMIN_SQL_DIR, 'create_db.sql')) if not res: raise Exception('Failed to create new database and user! Exit code: %i' % res) print('Creating database extensions...') res = db.run_sql_script_without_transaction(os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql')) # Don't raise an exception if the extension already exists application = webserver.create_app() with application.app_context(): print('Creating schema...') db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_schema.sql')) print('Creating tables...') db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql')) print('Creating primary and foreign keys...') db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql')) db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql')) print('Creating indexes...') db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql')) print("Done!")
def setUp(self): self.config = config db.init_db_connection(config.SQLALCHEMY_DATABASE_URI) self.reset_db()
def import_musicbrainz_rows(musicbrainz_db_uri, dry_run=True, delete=False): musicbrainz_db.init_db_engine(musicbrainz_db_uri) db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI']) import_count = 0 already_imported = 0 not_found = 0 deleted = 0 if not dry_run: update_row_ids_for_exceptions() users = db_user.get_all_users() with musicbrainz_db.engine.connect() as mb_connection: with db.engine.connect() as connection: for user in users: if user.get('musicbrainz_row_id') is not None: already_imported += 1 continue name = user['musicbrainz_id'] result = mb_connection.execute( sqlalchemy.text(""" SELECT id FROM editor WHERE LOWER(name) = LOWER(:name) """), { 'name': name, }) musicbrainz_row_id = None if result.rowcount > 0: musicbrainz_row_id = result.fetchone()['id'] import_count += 1 else: print( 'No user with specified username in the MusicBrainz db: %s' % name) if delete: print('Deleting user %s' % name) try: delete_user(user) except NotFound: print('User %s not found in LB...' % name) not_found += 1 continue if not dry_run: connection.execute( sqlalchemy.text(""" UPDATE "user" SET musicbrainz_row_id = :musicbrainz_row_id WHERE id = :id """), { 'musicbrainz_row_id': musicbrainz_row_id, 'id': user['id'], }) print('Inserted row_id %d for user %s' % (musicbrainz_row_id, name)) print('Total number of ListenBrainz users: %d' % len(users)) print( 'Total number of ListenBrainz users with already imported row ids: %d' % already_imported) print( 'Total number of ListenBrainz users whose row ids can be imported: %d' % import_count) print('Total number of ListenBrainz users not found in MusicBrainz: %d' % not_found) print('Total number of ListenBrainz users deleted from MusicBrainz: %d' % deleted)
def gen_app(config_path=None, debug=None): """ Generate a Flask app for LB with all configurations done and connections established. In the Flask app returned, blueprints are not registered. """ app = CustomFlask( import_name=__name__, use_flask_uuid=True, ) print("Starting metabrainz service with %s environment." % deploy_env) # Load configuration files: If we're running under a docker deployment, wait until config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'config.py') if deploy_env: print("Checking if consul template generated config file exists: %s" % config_file) for i in range(CONSUL_CONFIG_FILE_RETRY_COUNT): if not os.path.exists(config_file): sleep(1) if not os.path.exists(config_file): print( "No configuration file generated yet. Retried %d times, exiting." % CONSUL_CONFIG_FILE_RETRY_COUNT) sys.exit(-1) print("loading consul config file %s)" % config_file) app.config.from_pyfile(config_file) else: app.config.from_pyfile(config_file) if debug is not None: app.debug = debug # initialize Flask-DebugToolbar if the debug option is True if app.debug and app.config['SECRET_KEY']: app.init_debug_toolbar() # Output config values and some other info print('Configuration values are as follows: ') print(pprint.pformat(app.config, indent=4)) try: with open('.git-version') as git_version_file: print('Running on git commit: %s', git_version_file.read().strip()) except IOError as e: print('Unable to retrieve git commit. Error: %s', str(e)) # Logging app.init_loggers(file_config=app.config.get('LOG_FILE'), email_config=app.config.get('LOG_EMAIL'), sentry_config=app.config.get('LOG_SENTRY')) # Redis connection create_redis(app) # Influx connection create_influx(app) # RabbitMQ connection create_rabbitmq(app) # Database connection from listenbrainz import db db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI']) from listenbrainz.webserver.external import messybrainz messybrainz.init_db_connection( app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI']) # OAuth from listenbrainz.webserver.login import login_manager, provider login_manager.init_app(app) provider.init(app.config['MUSICBRAINZ_CLIENT_ID'], app.config['MUSICBRAINZ_CLIENT_SECRET']) # Error handling from listenbrainz.webserver.errors import init_error_handlers init_error_handlers(app) from listenbrainz.webserver import rate_limiter @app.after_request def after_request_callbacks(response): return rate_limiter.inject_x_rate_headers(response) # Template utilities app.jinja_env.add_extension('jinja2.ext.do') from listenbrainz.webserver import utils app.jinja_env.filters['date'] = utils.reformat_date app.jinja_env.filters['datetime'] = utils.reformat_datetime return app
def gen_app(config_path=None, debug=None): """ Generate a Flask app for LB with all configurations done and connections established. In the Flask app returned, blueprints are not registered. """ app = CustomFlask( import_name=__name__, use_flask_uuid=True, ) load_config(app) if debug is not None: app.debug = debug # initialize Flask-DebugToolbar if the debug option is True if app.debug and app.config['SECRET_KEY']: app.init_debug_toolbar() # Logging app.init_loggers(file_config=app.config.get('LOG_FILE'), email_config=app.config.get('LOG_EMAIL'), sentry_config=app.config.get('LOG_SENTRY')) # Redis connection create_redis(app) # Influx connection create_influx(app) # RabbitMQ connection try: create_rabbitmq(app) except ConnectionError: app.logger.critical("RabbitMQ service is not up!", exc_info=True) # Database connection from listenbrainz import db db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI']) from listenbrainz.webserver.external import messybrainz messybrainz.init_db_connection( app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI']) if app.config['MB_DATABASE_URI']: from brainzutils import musicbrainz_db musicbrainz_db.init_db_engine(app.config['MB_DATABASE_URI']) # OAuth from listenbrainz.webserver.login import login_manager, provider login_manager.init_app(app) provider.init(app.config['MUSICBRAINZ_CLIENT_ID'], app.config['MUSICBRAINZ_CLIENT_SECRET']) # Error handling from listenbrainz.webserver.errors import init_error_handlers init_error_handlers(app) from listenbrainz.webserver import rate_limiter @app.after_request def after_request_callbacks(response): return rate_limiter.inject_x_rate_headers(response) # Template utilities app.jinja_env.add_extension('jinja2.ext.do') from listenbrainz.webserver import utils app.jinja_env.filters['date'] = utils.reformat_date app.jinja_env.filters['datetime'] = utils.reformat_datetime return app
def recalculate_all_user_data(): timescale.init_db_connection(config.SQLALCHEMY_TIMESCALE_URI) db.init_db_connection(config.SQLALCHEMY_DATABASE_URI) init_cache(host=config.REDIS_HOST, port=config.REDIS_PORT, namespace=config.REDIS_NAMESPACE) # Find the created timestamp of the last listen query = "SELECT max(created) FROM listen WHERE created > :date" try: with timescale.engine.connect() as connection: result = connection.execute(sqlalchemy.text(query), date=datetime.now() - timedelta(weeks=4)) row = result.fetchone() last_created_ts = row[0] except psycopg2.OperationalError as e: logger.error("Cannot query ts to fetch latest listen." % str(e), exc_info=True) raise logger.info("Last created timestamp: " + str(last_created_ts)) # Select a list of users user_list = [] query = 'SELECT musicbrainz_id FROM "user"' try: with db.engine.connect() as connection: result = connection.execute(sqlalchemy.text(query)) for row in result: user_list.append(row[0]) except psycopg2.OperationalError as e: logger.error("Cannot query db to fetch user list." % str(e), exc_info=True) raise logger.info("Fetched %d users. Setting empty cache entries." % len(user_list)) # Reset the timestamps and listen counts to 0 for all users for user_name in user_list: cache.set(REDIS_USER_LISTEN_COUNT + user_name, 0, expirein=0, encode=False) cache.set(REDIS_USER_LISTEN_COUNT + user_name, 0, expirein=0, encode=False) cache.set(REDIS_USER_TIMESTAMPS + user_name, "0,0", expirein=0) # Tabulate all of the listen counts/timestamps for all users logger.info("Scan the whole listen table...") listen_counts = defaultdict(int) user_timestamps = {} query = "SELECT listened_at, user_name FROM listen where created <= :ts" try: with timescale.engine.connect() as connection: result = connection.execute(sqlalchemy.text(query), ts=last_created_ts) for row in result: ts = row[0] user_name = row[1] if user_name not in user_timestamps: user_timestamps[user_name] = [ts, ts] else: if ts > user_timestamps[user_name][1]: user_timestamps[user_name][1] = ts if ts < user_timestamps[user_name][0]: user_timestamps[user_name][0] = ts listen_counts[user_name] += 1 except psycopg2.OperationalError as e: logger.error("Cannot query db to fetch user list." % str(e), exc_info=True) raise logger.info("Setting updated cache entries.") # Set the timestamps and listen counts for all users for user_name in user_list: try: cache.increment(REDIS_USER_LISTEN_COUNT + user_name, amount=listen_counts[user_name]) except KeyError: pass try: tss = cache.get(REDIS_USER_TIMESTAMPS + user_name) (min_ts, max_ts) = tss.split(",") min_ts = int(min_ts) max_ts = int(max_ts) if min_ts and min_ts < user_timestamps[user_name][0]: user_timestamps[user_name][0] = min_ts if max_ts and max_ts > user_timestamps[user_name][1]: user_timestamps[user_name][1] = max_ts cache.set( REDIS_USER_TIMESTAMPS + user_name, "%d,%d" % (user_timestamps[user_name][0], user_timestamps[user_name][1]), expirein=0) except KeyError: pass
def create_app(debug=None): """ Generate a Flask app for LB with all configurations done and connections established. In the Flask app returned, blueprints are not registered. """ app = CustomFlask( import_name=__name__, use_flask_uuid=True, ) load_config(app) if debug is not None: app.debug = debug # As early as possible, if debug is True, set the log level of our 'listenbrainz' logger to DEBUG # to prevent flask from creating a new log handler if app.debug: logger = logging.getLogger('listenbrainz') logger.setLevel(logging.DEBUG) # initialize Flask-DebugToolbar if the debug option is True if app.debug and app.config['SECRET_KEY']: app.init_debug_toolbar() sentry_config = app.config.get('LOG_SENTRY') if sentry_config: sentry.init_sentry(**sentry_config) # Initialize BU cache and metrics cache.init(host=app.config['REDIS_HOST'], port=app.config['REDIS_PORT'], namespace=app.config['REDIS_NAMESPACE']) metrics.init("listenbrainz") # Database connections from listenbrainz import db from listenbrainz.db import timescale as ts from listenbrainz import messybrainz as msb db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI']) ts.init_db_connection(app.config['SQLALCHEMY_TIMESCALE_URI']) msb.init_db_connection(app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI']) # Redis connection from listenbrainz.webserver.redis_connection import init_redis_connection init_redis_connection(app.logger) # Timescale connection from listenbrainz.webserver.timescale_connection import init_timescale_connection init_timescale_connection(app) # RabbitMQ connection from listenbrainz.webserver.rabbitmq_connection import init_rabbitmq_connection try: init_rabbitmq_connection(app) except ConnectionError: app.logger.critical("RabbitMQ service is not up!", exc_info=True) if app.config['MB_DATABASE_URI']: from brainzutils import musicbrainz_db musicbrainz_db.init_db_engine(app.config['MB_DATABASE_URI']) # OAuth from listenbrainz.webserver.login import login_manager, provider login_manager.init_app(app) provider.init(app.config['MUSICBRAINZ_CLIENT_ID'], app.config['MUSICBRAINZ_CLIENT_SECRET']) # Error handling from listenbrainz.webserver.errors import init_error_handlers init_error_handlers(app) from brainzutils.ratelimit import inject_x_rate_headers, set_user_validation_function set_user_validation_function(check_ratelimit_token_whitelist) @app.after_request def after_request_callbacks(response): return inject_x_rate_headers(response) # Template utilities app.jinja_env.add_extension('jinja2.ext.do') from listenbrainz.webserver import utils app.jinja_env.filters['date'] = utils.reformat_date app.jinja_env.filters['datetime'] = utils.reformat_datetime return app
def setUp(self): self.config = config db.init_db_connection(config.SQLALCHEMY_DATABASE_URI) self.reset_db()