def test_import_postgres_db(self): # create a user db_user.create(1, 'test_user') user_count = db_user.get_user_count() self.assertEqual(user_count, 1) # do a db dump and reset the db private_dump, public_dump = db_dump.dump_postgres_db(self.tempdir) self.reset_db() user_count = db_user.get_user_count() self.assertEqual(user_count, 0) # import the dump db_dump.import_postgres_dump(private_dump, public_dump) user_count = db_user.get_user_count() self.assertEqual(user_count, 1) # reset again, and use more threads to import self.reset_db() user_count = db_user.get_user_count() self.assertEqual(user_count, 0) db_dump.import_postgres_dump(private_dump, public_dump, threads=2) user_count = db_user.get_user_count() self.assertEqual(user_count, 1)
def test_dump_recording_feedback(self): # create a user with self.app.app_context(): one_id = db_user.create(1, 'test_user') user_count = db_user.get_user_count() self.assertEqual(user_count, 1) # insert a feedback record feedback = Feedback( user_id=one_id, recording_msid="d23f4719-9212-49f0-ad08-ddbfbfc50d6f", score=1) db_feedback.insert(feedback) # do a db dump and reset the db private_dump, private_ts_dump, public_dump, public_ts_dump = db_dump.dump_postgres_db( self.tempdir) self.reset_db() user_count = db_user.get_user_count() self.assertEqual(user_count, 0) self.assertEqual( db_feedback.get_feedback_count_for_user(user_id=one_id), 0) # import the dump and check the records are inserted db_dump.import_postgres_dump(private_dump, None, public_dump, None) user_count = db_user.get_user_count() self.assertEqual(user_count, 1) dumped_feedback = db_feedback.get_feedback_for_user(user_id=one_id, limit=1, offset=0) self.assertEqual(len(dumped_feedback), 1) self.assertEqual(dumped_feedback[0].user_id, feedback.user_id) self.assertEqual(dumped_feedback[0].recording_msid, feedback.recording_msid) self.assertEqual(dumped_feedback[0].score, feedback.score) # reset again, and use more threads to import self.reset_db() user_count = db_user.get_user_count() self.assertEqual(user_count, 0) dumped_feedback = [] db_dump.import_postgres_dump(private_dump, None, public_dump, None, threads=2) user_count = db_user.get_user_count() self.assertEqual(user_count, 1) dumped_feedback = db_feedback.get_feedback_for_user(user_id=one_id, limit=1, offset=0) self.assertEqual(len(dumped_feedback), 1) self.assertEqual(dumped_feedback[0].user_id, feedback.user_id) self.assertEqual(dumped_feedback[0].recording_msid, feedback.recording_msid) self.assertEqual(dumped_feedback[0].score, feedback.score)
def test_import_postgres_db(self): # create a user with self.app.app_context(): one_id = db_user.create(1, 'test_user') user_count = db_user.get_user_count() self.assertEqual(user_count, 1) # do a db dump and reset the db private_dump, private_ts_dump, public_dump, public_ts_dump = db_dump.dump_postgres_db( self.tempdir) self.reset_db() user_count = db_user.get_user_count() self.assertEqual(user_count, 0) # import the dump db_dump.import_postgres_dump(private_dump, None, public_dump, None) user_count = db_user.get_user_count() self.assertEqual(user_count, 1) # reset again, and use more threads to import self.reset_db() user_count = db_user.get_user_count() self.assertEqual(user_count, 0) db_dump.import_postgres_dump(private_dump, None, public_dump, None, threads=2) user_count = db_user.get_user_count() self.assertEqual(user_count, 1) two_id = db_user.create(2, 'vnskprk') self.assertGreater(two_id, one_id)
def test_import_postgres_db(self): # create a user db_user.create('test_user') user_count = db_user.get_user_count() self.assertEqual(user_count, 1) # do a db dump and reset the db location = db_dump.dump_postgres_db(self.tempdir) self.reset_db() user_count = db_user.get_user_count() self.assertEqual(user_count, 0) # import the dump db_dump.import_postgres_dump(location) user_count = db_user.get_user_count() self.assertEqual(user_count, 1) # reset again, and use more threads to import self.reset_db() user_count = db_user.get_user_count() self.assertEqual(user_count, 0) db_dump.import_postgres_dump(location, threads=2) user_count = db_user.get_user_count() self.assertEqual(user_count, 1)
def test_import_postgres_db(self): # create a user with self.app.app_context(): one_id = db_user.create(1, 'test_user') user_count = db_user.get_user_count() self.assertEqual(user_count, 1) # do a db dump and reset the db private_dump, public_dump = db_dump.dump_postgres_db(self.tempdir) self.reset_db() user_count = db_user.get_user_count() self.assertEqual(user_count, 0) # import the dump db_dump.import_postgres_dump(private_dump, public_dump) user_count = db_user.get_user_count() self.assertEqual(user_count, 1) # reset again, and use more threads to import self.reset_db() user_count = db_user.get_user_count() self.assertEqual(user_count, 0) db_dump.import_postgres_dump(private_dump, public_dump, threads=2) user_count = db_user.get_user_count() self.assertEqual(user_count, 1) two_id = db_user.create(2, 'vnskprk') self.assertGreater(two_id, one_id)
def import_dump(private_archive, public_archive, listen_archive, threads): """ Import a ListenBrainz dump into the database. Note: This method tries to import the private db dump first, followed by the public db dump. However, in absence of a private dump, it imports sanitized versions of the user table in the public dump in order to satisfy foreign key constraints. Then it imports the listen dump. Args: private_archive (str): the path to the ListenBrainz private dump to be imported public_archive (str): the path to the ListenBrainz public dump to be imported listen_archive (str): the path to the ListenBrainz listen dump archive to be imported threads (int): the number of threads to use during decompression, defaults to 1 """ if not private_archive and not public_archive and not listen_archive: print('You need to enter a path to the archive(s) to import!') sys.exit(1) app = create_app() with app.app_context(): db_dump.import_postgres_dump(private_archive, public_archive, threads) from listenbrainz.webserver.timescale_connection import _ts as ls try: ls.import_listens_dump(listen_archive, threads) except psycopg2.OperationalError as e: current_app.logger.critical('OperationalError while trying to import data: %s', str(e), exc_info=True) raise except IOError as e: current_app.logger.critical('IOError while trying to import data: %s', str(e), exc_info=True) raise except Exception as e: current_app.logger.critical('Unexpected error while importing data: %s', str(e), exc_info=True) raise
def import_dump(private_archive, public_archive, listen_archive, threads): """ Import a ListenBrainz dump into the database. Note: This method tries to import the private db dump first, followed by the public db dump. However, in absence of a private dump, it imports sanitized versions of the user table in the public dump in order to satisfy foreign key constraints. Then it imports the listen dump. Args: private_archive (str): the path to the ListenBrainz private dump to be imported public_archive (str): the path to the ListenBrainz public dump to be imported listen_archive (str): the path to the ListenBrainz listen dump archive to be imported threads (int): the number of threads to use during decompression, defaults to 1 """ if not private_archive and not public_archive and not listen_archive: print('You need to enter a path to the archive(s) to import!') sys.exit(1) app = create_app() with app.app_context(): db_dump.import_postgres_dump(private_archive, public_archive, threads) ls = init_influx_connection( current_app.logger, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], }) try: ls.import_listens_dump(listen_archive, threads) except IOError as e: current_app.logger.critical( 'IOError while trying to import data into Influx: %s', str(e), exc_info=True) raise except InfluxDBClientError as e: current_app.logger.critical( 'Error while sending data to Influx: %s', str(e), exc_info=True) raise except InfluxDBServerError as e: current_app.logger.critical( 'InfluxDB Server Error while importing data: %s', str(e), exc_info=True) raise except Exception as e: current_app.logger.critical( 'Unexpected error while importing data: %s', str(e), exc_info=True) raise
def import_db(location, threads=None): """ Import a ListenBrainz PostgreSQL dump into the PostgreSQL database. Note: This method tries to import the private dump first, followed by the statistics dump. However, in absence of a private dump, it imports sanitized versions of the user table in the statistics dump in order to satisfy foreign key constraints. Args: location (str): path to the directory which contains the private and the stats dump threads (int): the number of threads to use during decompression, defaults to 1 """ db.init_db_connection(config.SQLALCHEMY_DATABASE_URI) db_dump.import_postgres_dump(location, threads)
def import_dump(private_archive, public_archive, listen_archive, threads): """ Import a ListenBrainz dump into the database. Note: This method tries to import the private db dump first, followed by the public db dump. However, in absence of a private dump, it imports sanitized versions of the user table in the public dump in order to satisfy foreign key constraints. Then it imports the listen dump. Args: private_archive (str): the path to the ListenBrainz private dump to be imported public_archive (str): the path to the ListenBrainz public dump to be imported listen_archive (str): the path to the ListenBrainz listen dump archive to be imported threads (int): the number of threads to use during decompression, defaults to 1 """ if not private_archive and not public_archive and not listen_archive: print('You need to enter a path to the archive(s) to import!') sys.exit(1) app = create_app() with app.app_context(): db_dump.import_postgres_dump(private_archive, public_archive, threads) ls = init_influx_connection(current_app.logger, { 'REDIS_HOST': current_app.config['REDIS_HOST'], 'REDIS_PORT': current_app.config['REDIS_PORT'], 'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'], 'INFLUX_HOST': current_app.config['INFLUX_HOST'], 'INFLUX_PORT': current_app.config['INFLUX_PORT'], 'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'], }) try: ls.import_listens_dump(listen_archive, threads) except IOError as e: current_app.logger.critical('IOError while trying to import data into Influx: %s', str(e), exc_info=True) raise except InfluxDBClientError as e: current_app.logger.critical('Error while sending data to Influx: %s', str(e), exc_info=True) raise except InfluxDBServerError as e: current_app.logger.critical('InfluxDB Server Error while importing data: %s', str(e), exc_info=True) raise except Exception as e: current_app.logger.critical('Unexpected error while importing data: %s', str(e), exc_info=True) raise