def setUp(self): super(TestAPICompatUserClass, self).setUp() self.log = logging.getLogger(__name__) self.logstore = init_influx_connection( self.log, { 'INFLUX_HOST': config.INFLUX_HOST, 'INFLUX_PORT': config.INFLUX_PORT, 'INFLUX_DB_NAME': config.INFLUX_DB_NAME, 'REDIS_HOST': config.REDIS_HOST, 'REDIS_PORT': config.REDIS_PORT, }) # Create a user uid = db_user.create("test_api_compat_user") self.assertIsNotNone(db_user.get(uid)) with db.engine.connect() as connection: result = connection.execute( text(""" SELECT * FROM "user" WHERE id = :id """), { "id": uid, }) row = result.fetchone() self.user = User(row['id'], row['created'], row['musicbrainz_id'], row['auth_token']) # Insert some listens date = datetime(2015, 9, 3, 0, 0, 0) self.log.info("Inserting test data...") test_data = generate_data(date, 100, self.user.name) self.logstore.insert(test_data) self.log.info("Test data inserted")
def create(location, threads): """ Create a ListenBrainz data dump which includes a private dump, a statistics dump and a dump of the actual listens from InfluxDB Args: location (str): path to the directory where the dump should be made threads (int): the number of threads to be used while compression """ app = create_app() with app.app_context(): ls = init_influx_connection(current_app.logger, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], }) time_now = datetime.today() dump_path = os.path.join(location, 'listenbrainz-dump-{time}'.format(time=time_now.strftime('%Y%m%d-%H%M%S'))) create_path(dump_path) db_dump.dump_postgres_db(dump_path, time_now, threads) ls.dump_listens(dump_path, time_now, threads) try: write_hashes(dump_path) except IOError as e: current_app.logger.error('Unable to create hash files! Error: %s', str(e), exc_info=True) return current_app.logger.info('Dumps created and hashes written at %s' % dump_path)
def setUp(self): ServerTestCase.setUp(self) DatabaseTestCase.setUp(self) self.log = logging.getLogger(__name__) self.influx = InfluxDBClient( host=current_app.config['INFLUX_HOST'], port=current_app.config['INFLUX_PORT'], database=current_app.config['INFLUX_DB_NAME'], ) self.influx.query('''create database %s''' % current_app.config['INFLUX_DB_NAME']) self.logstore = init_influx_connection(self.log, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], }) user = db_user.get_or_create(1, 'iliekcomputers') self.user = User.from_dbrow(user) weirduser = db_user.get_or_create(2, 'weird\\user name') self.weirduser = User.from_dbrow(weirduser)
def create(location, threads): """ Create a ListenBrainz data dump which includes a private dump, a statistics dump and a dump of the actual listens from InfluxDB Args: location (str): path to the directory where the dump should be made threads (int): the number of threads to be used while compression """ db.init_db_connection(config.SQLALCHEMY_DATABASE_URI) ls = init_influx_connection(log, { 'REDIS_HOST': config.REDIS_HOST, 'REDIS_PORT': config.REDIS_PORT, 'REDIS_NAMESPACE': config.REDIS_NAMESPACE, 'INFLUX_HOST': config.INFLUX_HOST, 'INFLUX_PORT': config.INFLUX_PORT, 'INFLUX_DB_NAME': config.INFLUX_DB_NAME, }) time_now = datetime.today() dump_path = os.path.join(location, 'listenbrainz-dump-{time}'.format(time=time_now.strftime('%Y%m%d-%H%M%S'))) create_path(dump_path) db_dump.dump_postgres_db(dump_path, time_now, threads) ls.dump_listens(dump_path, time_now, threads) try: write_hashes(dump_path) except IOError as e: log.error('Unable to create hash files! Error: %s', str(e)) return log.info('Dumps created and hashes written at %s' % dump_path)
def setUp(self): ServerTestCase.setUp(self) DatabaseTestCase.setUp(self) self.log = logging.getLogger(__name__) self.influx = InfluxDBClient( host=current_app.config['INFLUX_HOST'], port=current_app.config['INFLUX_PORT'], database=current_app.config['INFLUX_DB_NAME'], ) self.influx.query('''create database %s''' % current_app.config['INFLUX_DB_NAME']) self.logstore = init_influx_connection( self.log, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], }) user = db_user.get_or_create(1, 'iliekcomputers') db_user.agree_to_gdpr(user['musicbrainz_id']) self.user = User.from_dbrow(user) weirduser = db_user.get_or_create(2, 'weird\\user name') self.weirduser = User.from_dbrow(weirduser)
def create_spark_dump(location, threads): with create_app().app_context(): ls = init_influx_connection( current_app.logger, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], }) time_now = datetime.today() dump_path = os.path.join( location, 'listenbrainz-spark-dump-{time}'.format( time=time_now.strftime('%Y%m%d-%H%M%S'))) create_path(dump_path) ls.dump_listens(dump_path, time_now, threads, spark_format=True) try: write_hashes(dump_path) except IOError as e: current_app.logger.error('Unable to create hash files! Error: %s', str(e), exc_info=True) return current_app.logger.info('Dump created and hash written at %s', dump_path)
def setUp(self): super(TestAPICompatUserClass, self).setUp() self.log = logging.getLogger(__name__) self.logstore = init_influx_connection(self.log, { 'INFLUX_HOST': config.INFLUX_HOST, 'INFLUX_PORT': config.INFLUX_PORT, 'INFLUX_DB_NAME': config.INFLUX_DB_NAME, 'REDIS_HOST': config.REDIS_HOST, 'REDIS_PORT': config.REDIS_PORT, 'REDIS_NAMESPACE': config.REDIS_NAMESPACE, }) # Create a user uid = db_user.create(1, "test_api_compat_user") self.assertIsNotNone(db_user.get(uid)) with db.engine.connect() as connection: result = connection.execute(text(""" SELECT * FROM "user" WHERE id = :id """), { "id": uid, }) row = result.fetchone() self.user = User(row['id'], row['created'], row['musicbrainz_id'], row['auth_token'])
def find_users(self): with self.app.app_context(): self.ls = init_influx_connection( current_app.logger, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], }) for _ in range(CONNECTION_RETRY_COUNT): try: users = db_user.get_all_users() break except DatabaseError as e: current_app.logger.error( 'Error while getting users list: %s', str(e), exc_info=True) time.sleep(1) else: current_app.logger.critical( "Cannot connect to PostgreSQL, exiting...") raise DatabaseError("Cannot connect to PostgreSQL, exiting") return [ user['musicbrainz_id'] for user in users if self.condition(user['musicbrainz_id']) ]
def create(location, threads): """ Create a ListenBrainz data dump which includes a private dump, a statistics dump and a dump of the actual listens from InfluxDB Args: location (str): path to the directory where the dump should be made threads (int): the number of threads to be used while compression """ app = create_app() with app.app_context(): ls = init_influx_connection( current_app.logger, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], }) time_now = datetime.today() dump_path = os.path.join( location, 'listenbrainz-dump-{time}'.format( time=time_now.strftime('%Y%m%d-%H%M%S'))) create_path(dump_path) db_dump.dump_postgres_db(dump_path, time_now, threads) ls.dump_listens(dump_path, time_now, threads) try: write_hashes(dump_path) except IOError as e: current_app.logger.error('Unable to create hash files! Error: %s', str(e), exc_info=True) return current_app.logger.info('Dumps created and hashes written at %s' % dump_path)
def setUp(self): super(TestAPICompatUserClass, self).setUp() self.log = logging.getLogger(__name__) self.logstore = init_influx_connection( self.log, { 'INFLUX_HOST': config.INFLUX_HOST, 'INFLUX_PORT': config.INFLUX_PORT, 'INFLUX_DB_NAME': config.INFLUX_DB_NAME, 'REDIS_HOST': config.REDIS_HOST, 'REDIS_PORT': config.REDIS_PORT, 'REDIS_NAMESPACE': config.REDIS_NAMESPACE, }) # Create a user uid = db_user.create("test_api_compat_user") self.assertIsNotNone(db_user.get(uid)) with db.engine.connect() as connection: result = connection.execute( text(""" SELECT * FROM "user" WHERE id = :id """), { "id": uid, }) row = result.fetchone() self.user = User(row['id'], row['created'], row['musicbrainz_id'], row['auth_token'])
def import_dump(private_archive, public_archive, listen_archive, threads): """ Import a ListenBrainz dump into the database. Note: This method tries to import the private db dump first, followed by the public db dump. However, in absence of a private dump, it imports sanitized versions of the user table in the public dump in order to satisfy foreign key constraints. Then it imports the listen dump. Args: private_archive (str): the path to the ListenBrainz private dump to be imported public_archive (str): the path to the ListenBrainz public dump to be imported listen_archive (str): the path to the ListenBrainz listen dump archive to be imported threads (int): the number of threads to use during decompression, defaults to 1 """ if not private_archive and not public_archive and not listen_archive: print('You need to enter a path to the archive(s) to import!') sys.exit(1) app = create_app() with app.app_context(): db_dump.import_postgres_dump(private_archive, public_archive, threads) ls = init_influx_connection( current_app.logger, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], }) try: ls.import_listens_dump(listen_archive, threads) except IOError as e: current_app.logger.critical( 'IOError while trying to import data into Influx: %s', str(e), exc_info=True) raise except InfluxDBClientError as e: current_app.logger.critical( 'Error while sending data to Influx: %s', str(e), exc_info=True) raise except InfluxDBServerError as e: current_app.logger.critical( 'InfluxDB Server Error while importing data: %s', str(e), exc_info=True) raise except Exception as e: current_app.logger.critical( 'Unexpected error while importing data: %s', str(e), exc_info=True) raise
def create_incremental(location, threads, dump_id): app = create_app() with app.app_context(): ls = init_influx_connection( current_app.logger, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], }) if dump_id is None: end_time = datetime.now() dump_id = db_dump.add_dump_entry(int(end_time.strftime('%s'))) else: dump_entry = db_dump.get_dump_entry(dump_id) if dump_entry is None: current_app.logger.error("No dump with ID %d found, exiting!", dump_id) sys.exit(-1) end_time = dump_entry['created'] prev_dump_entry = db_dump.get_dump_entry(dump_id - 1) if prev_dump_entry is None: # incremental dumps must have a previous dump in the series current_app.logger.error( "Invalid dump ID %d, could not find previous dump", dump_id) sys.exit(-1) start_time = prev_dump_entry['created'] current_app.logger.info("Dumping data from %s to %s", start_time, end_time) dump_path = os.path.join( location, 'listenbrainz-dump-{dump_id}-{time}-incremental'.format( dump_id=dump_id, time=end_time.strftime('%Y%m%d-%H%M%S'))) create_path(dump_path) ls.dump_listens(dump_path, dump_id=dump_id, start_time=start_time, end_time=end_time, threads=threads, spark_format=False) ls.dump_listens(dump_path, dump_id=dump_id, start_time=start_time, end_time=end_time, threads=threads, spark_format=True) try: write_hashes(dump_path) except IOError as e: current_app.logger.error('Unable to create hash files! Error: %s', str(e), exc_info=True) return current_app.logger.info('Dumps created and hashes written at %s' % dump_path)
def create_influx(app): from listenbrainz.webserver.influx_connection import init_influx_connection return init_influx_connection(app.logger, { 'INFLUX_HOST': app.config['INFLUX_HOST'], 'INFLUX_PORT': app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': app.config['INFLUX_DB_NAME'], 'REDIS_HOST': app.config['REDIS_HOST'], 'REDIS_PORT': app.config['REDIS_PORT'], })
def setUp(self): super(TestInfluxListenStore, self).setUp() self.log = logging.getLogger(__name__) self.logstore = init_influx_connection( self.log, { 'REDIS_HOST': config.REDIS_HOST, 'REDIS_PORT': config.REDIS_PORT, 'INFLUX_HOST': config.INFLUX_HOST, 'INFLUX_PORT': config.INFLUX_PORT, 'INFLUX_DB_NAME': config.INFLUX_DB_NAME, }) self.testuser_id = db_user.create("test") user = db_user.get(self.testuser_id) print(user) self.testuser_name = db_user.get(self.testuser_id)['musicbrainz_id']
def setUp(self): super(TestInfluxListenStore, self).setUp() self.log = logging.getLogger(__name__) # In order to do counting correctly, we need a clean DB to start with self.reset_influx_db() self.logstore = init_influx_connection(self.log, { 'REDIS_HOST': config.REDIS_HOST, 'REDIS_PORT': config.REDIS_PORT, 'INFLUX_HOST': config.INFLUX_HOST, 'INFLUX_PORT': config.INFLUX_PORT, 'INFLUX_DB_NAME': config.INFLUX_DB_NAME, }) self.testuser_id = db_user.create("test") self.testuser_name = db_user.get(self.testuser_id)['musicbrainz_id']
def setUp(self): super().setUp() app = create_app() self.tempdir = tempfile.mkdtemp() self.runner = CliRunner() self.listenstore = init_influx_connection( logging.getLogger(__name__), { 'REDIS_HOST': app.config['REDIS_HOST'], 'REDIS_PORT': app.config['REDIS_PORT'], 'REDIS_NAMESPACE': app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': app.config['INFLUX_HOST'], 'INFLUX_PORT': app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': app.config['INFLUX_DB_NAME'], }) self.user_id = db_user.create(1, 'iliekcomputers') self.user_name = db_user.get(self.user_id)['musicbrainz_id']
def import_dump(private_archive, public_archive, listen_archive, threads): """ Import a ListenBrainz dump into the database. Note: This method tries to import the private db dump first, followed by the public db dump. However, in absence of a private dump, it imports sanitized versions of the user table in the public dump in order to satisfy foreign key constraints. Then it imports the listen dump. Args: private_archive (str): the path to the ListenBrainz private dump to be imported public_archive (str): the path to the ListenBrainz public dump to be imported listen_archive (str): the path to the ListenBrainz listen dump archive to be imported threads (int): the number of threads to use during decompression, defaults to 1 """ if not private_archive and not public_archive and not listen_archive: print('You need to enter a path to the archive(s) to import!') sys.exit(1) app = create_app() with app.app_context(): db_dump.import_postgres_dump(private_archive, public_archive, threads) ls = init_influx_connection(current_app.logger, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], }) try: ls.import_listens_dump(listen_archive, threads) except IOError as e: current_app.logger.critical('IOError while trying to import data into Influx: %s', str(e), exc_info=True) raise except InfluxDBClientError as e: current_app.logger.critical('Error while sending data to Influx: %s', str(e), exc_info=True) raise except InfluxDBServerError as e: current_app.logger.critical('InfluxDB Server Error while importing data: %s', str(e), exc_info=True) raise except Exception as e: current_app.logger.critical('Unexpected error while importing data: %s', str(e), exc_info=True) raise
def setUp(self): super(TestInfluxListenStore, self).setUp() self.log = logging.getLogger(__name__) # In order to do counting correctly, we need a clean DB to start with self.reset_influx_db() self.logstore = init_influx_connection(self.log, { 'REDIS_HOST': config.REDIS_HOST, 'REDIS_PORT': config.REDIS_PORT, 'REDIS_NAMESPACE': config.REDIS_NAMESPACE, 'INFLUX_HOST': config.INFLUX_HOST, 'INFLUX_PORT': config.INFLUX_PORT, 'INFLUX_DB_NAME': config.INFLUX_DB_NAME, }) self.testuser_id = db_user.create(1, "test") self.testuser_name = db_user.get(self.testuser_id)['musicbrainz_id']
def start(self): with self.app.app_context(): current_app.logger.info("Connecting to Influx...") self.ls = init_influx_connection( current_app.logger, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], }) current_app.logger.info("Done!") new_measurement_name = str(uuid.uuid4()) current_app.logger.info("Temporary destination measurement: %s", new_measurement_name) current_app.logger.info( "Copying listens from %s to temporary measurement...", self.user_name) self.copy_measurement(src=self.user_name, dest=new_measurement_name, apply_filter=True) current_app.logger.info("Done!") current_app.logger.info("Removing user measurement...") self.ls.delete(self.user_name) current_app.logger.info("Done!") current_app.logger.info( "Copying listens back from temporary measurement to %s...", self.user_name) self.copy_measurement(src=new_measurement_name, dest=self.user_name, apply_filter=False) current_app.logger.info("Done!") current_app.logger.info("Removing temporary measurement...") self.ls.delete(new_measurement_name) current_app.logger.info("Done!")
def import_listens(location=None, threads=None): """ Import a ListenBrainz listen dump into the Influx database. Args: location (str): path to the listenbrainz listen .tar.xz archive threads (int): the number of threads to use while decompressing, defaults to 1 """ if not location: print('No location given!') sys.exit(1) ls = init_influx_connection( log, { 'REDIS_HOST': config.REDIS_HOST, 'REDIS_PORT': config.REDIS_PORT, 'INFLUX_HOST': config.INFLUX_HOST, 'INFLUX_PORT': config.INFLUX_PORT, 'INFLUX_DB_NAME': config.INFLUX_DB_NAME, }) try: ls.import_listens_dump(location, threads) except IOError as e: log.error('IOError while trying to import data into Influx: %s', str(e)) raise except InfluxDBClientError as e: log.error('Error while sending data to Influx: %s', str(e)) raise except InfluxDBServerError as e: log.error('InfluxDB Server Error while importing data: %s', str(e)) raise except Exception as e: log.error('Unexpected error while importing data: %s', str(e)) raise
def setUp(self): ServerTestCase.setUp(self) DatabaseTestCase.setUp(self) self.user = db_user.get_or_create('iliekcomputers') self.weirduser = db_user.get_or_create('weird\\user name') self.log = logging.getLogger(__name__) self.influx = InfluxDBClient( host=current_app.config['INFLUX_HOST'], port=current_app.config['INFLUX_PORT'], database=current_app.config['INFLUX_DB_NAME'], ) self.influx.query('''create database %s''' % current_app.config['INFLUX_DB_NAME']) self.logstore = init_influx_connection( self.log, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], })
from listenbrainz.webserver import create_app from listenbrainz.webserver.views.api_tools import publish_data_to_queue from listenbrainz.listenstore import InfluxListenStore from listenbrainz.webserver.influx_connection import init_influx_connection from werkzeug.exceptions import NotFound import listenbrainz.db.user as db_user import logging import sqlalchemy app = create_app() influx = init_influx_connection( logging, { 'REDIS_HOST': app.config['REDIS_HOST'], 'REDIS_PORT': app.config['REDIS_PORT'], 'REDIS_NAMESPACE': app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': app.config['INFLUX_HOST'], 'INFLUX_PORT': app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': app.config['INFLUX_DB_NAME'], }) def update_row_ids_for_exceptions(): with musicbrainz_db.engine.connect() as mb_connection: with db.engine.connect() as connection: # 2106 - Fée Deuspi result = mb_connection.execute( sqlalchemy.text(""" SELECT id FROM editor WHERE name = 'Fée Deuspi'
def create_full(location, threads, dump_id, last_dump_id): """ Create a ListenBrainz data dump which includes a private dump, a statistics dump and a dump of the actual listens from InfluxDB Args: location (str): path to the directory where the dump should be made threads (int): the number of threads to be used while compression dump_id (int): the ID of the ListenBrainz data dump last_dump_id (bool): flag indicating whether to create a full dump from the last entry in the dump table """ app = create_app() with app.app_context(): ls = init_influx_connection( current_app.logger, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], }) if last_dump_id: all_dumps = db_dump.get_dump_entries() if len(all_dumps) == 0: current_app.logger.error( "Cannot create full dump with last dump's ID, no dump exists!" ) sys.exit(-1) dump_id = all_dumps[0]['id'] if dump_id is None: end_time = datetime.now() dump_id = db_dump.add_dump_entry(int(end_time.strftime('%s'))) else: dump_entry = db_dump.get_dump_entry(dump_id) if dump_entry is None: current_app.logger.error("No dump with ID %d found", dump_id) sys.exit(-1) end_time = dump_entry['created'] dump_path = os.path.join( location, 'listenbrainz-dump-{dump_id}-{time}-full'.format( dump_id=dump_id, time=end_time.strftime('%Y%m%d-%H%M%S'))) create_path(dump_path) db_dump.dump_postgres_db(dump_path, end_time, threads) ls.dump_listens(dump_path, dump_id=dump_id, end_time=end_time, threads=threads, spark_format=False) ls.dump_listens(dump_path, dump_id=dump_id, end_time=end_time, threads=threads, spark_format=True) try: write_hashes(dump_path) except IOError as e: current_app.logger.error('Unable to create hash files! Error: %s', str(e), exc_info=True) return current_app.logger.info('Dumps created and hashes written at %s' % dump_path)
from brainzutils import musicbrainz_db from listenbrainz.webserver import create_app from listenbrainz.webserver.views.api_tools import publish_data_to_queue from listenbrainz.listenstore import InfluxListenStore from listenbrainz.webserver.influx_connection import init_influx_connection from werkzeug.exceptions import NotFound import listenbrainz.db.user as db_user import logging import sqlalchemy app = create_app() influx = init_influx_connection(logging, { 'REDIS_HOST': app.config['REDIS_HOST'], 'REDIS_PORT': app.config['REDIS_PORT'], 'REDIS_NAMESPACE': app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': app.config['INFLUX_HOST'], 'INFLUX_PORT': app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': app.config['INFLUX_DB_NAME'], }) def update_row_ids_for_exceptions(): with musicbrainz_db.engine.connect() as mb_connection: with db.engine.connect() as connection: # 2106 - Fée Deuspi result = mb_connection.execute(sqlalchemy.text(""" SELECT id FROM editor WHERE name = 'Fée Deuspi' """))