def test_search_dump(self): """ Test to ensure 'True' is returned if appropriate dump is found and 'False' if it isn't found. """ self.assertTrue(import_utils.search_dump(4, "full", datetime.fromtimestamp(3))) self.assertFalse(import_utils.search_dump(4, "full", datetime.fromtimestamp(5))) self.assertFalse(import_utils.search_dump(5, "full", datetime.fromtimestamp(5))) self.assertTrue(import_utils.search_dump(4, "incremental", datetime.fromtimestamp(4))) self.assertFalse(import_utils.search_dump(4, "incremental", datetime.fromtimestamp(5)))
def test_insert_dump_data_file_missing(self): """ Test to ensure a file is created if it is missing. """ path_found = path_exists(self.path_) if path_found: delete_dir(self.path_, recursive=True) self.assertFalse(import_utils.search_dump(1, "full", datetime.fromtimestamp(1))) import_utils.insert_dump_data(1, "full", datetime.fromtimestamp(1)) self.assertTrue(import_utils.search_dump(1, "full", datetime.fromtimestamp(1)))
def test_insert_dump_data_update_date(self): """ Test to ensure date is updated if entry already exists. """ self.assertFalse( import_utils.search_dump(7, "incremental", datetime.fromtimestamp(9))) import_utils.insert_dump_data(7, "incremental", datetime.fromtimestamp(9)) self.assertTrue( import_utils.search_dump(7, "incremental", datetime.fromtimestamp(9)))
def test_insert_dump_data_update_date(self): """ Test to ensure date is updated if entry already exists. """ self.assertFalse( import_utils.search_dump(7, DumpType.INCREMENTAL, datetime.fromtimestamp(9))) import_utils.insert_dump_data(7, DumpType.INCREMENTAL, datetime.fromtimestamp(9)) self.assertTrue( import_utils.search_dump(7, DumpType.INCREMENTAL, datetime.fromtimestamp(9))) self.assertTrue( import_utils.search_dump(2, DumpType.INCREMENTAL, datetime.fromtimestamp(2)))
def import_newest_incremental_dump_handler(): imported_dumps = [] latest_full_dump = utils.get_latest_full_dump() if latest_full_dump is None: # If no prior full dump is present, just import the lates incremental dump imported_dumps.append( import_dump_to_hdfs('incremental', overwrite=False)) current_app.logger.warn( "No previous full dump found, importing latest incremental dump", exc_info=True) else: # Import all missing dumps from last full dump import dump_id = latest_full_dump["dump_id"] + 1 imported_at = latest_full_dump["imported_at"] while True: if not utils.search_dump(dump_id, 'incremental', imported_at): try: imported_dumps.append( import_dump_to_hdfs('incremental', False, dump_id)) except DumpNotFoundException: break except Exception as e: # Exit if any other error occurs during import current_app.logger.error( f"Error while importing incremental dump with ID {dump_id}: {e}", exc_info=True) break dump_id += 1 return [{ 'type': 'import_incremental_dump', 'imported_dump': imported_dumps, 'time': str(datetime.utcnow()), }]
def test_search_dump_file_missing(self): """ Test to ensure 'False' is returned if metadata file is missing. """ path_found = path_exists(self.path_) if path_found: delete_dir(self.path_, recursive=True) self.assertFalse(import_utils.search_dump(1, "full", datetime.fromtimestamp(1)))
def import_newest_incremental_dump_handler(): errors = [] imported_dumps = [] latest_full_dump = utils.get_latest_full_dump() if latest_full_dump is None: # If no prior full dump is present, just import the latest incremental dump imported_dumps.append(import_incremental_dump_to_hdfs(dump_id=None)) error_msg = "No previous full dump found, importing latest incremental dump" errors.append(error_msg) logger.warning(error_msg, exc_info=True) else: # Import all missing dumps from last full dump import start_id = latest_full_dump["dump_id"] + 1 imported_at = latest_full_dump["imported_at"] end_id = ListenbrainzDataDownloader().get_latest_dump_id(DumpType.INCREMENTAL) + 1 for dump_id in range(start_id, end_id, 1): if not utils.search_dump(dump_id, DumpType.INCREMENTAL, imported_at): try: imported_dumps.append(import_incremental_dump_to_hdfs(dump_id)) except Exception as e: # Skip current dump if any error occurs during import error_msg = f"Error while importing incremental dump with ID {dump_id}: {e}" errors.append(error_msg) logger.error(error_msg, exc_info=True) continue dump_id += 1 request_consumer.rc.ping() return [{ 'type': 'import_incremental_dump', 'imported_dump': imported_dumps, 'errors': errors, 'time': str(datetime.utcnow()), }]
def test_insert_dump_data(self): """ Test to ensure that data is inserted correctly. """ import_utils.insert_dump_data(9, DumpType.FULL, datetime.fromtimestamp(9)) self.assertTrue( import_utils.search_dump(9, DumpType.FULL, datetime.fromtimestamp(9)))
def test_search_dump(self): """ Test to ensure 'True' is returned if appropriate dump is found and 'False' if it isn't found. """ self.assertTrue( import_utils.search_dump(4, DumpType.FULL, datetime.fromtimestamp(3))) self.assertFalse( import_utils.search_dump(4, DumpType.FULL, datetime.fromtimestamp(5))) self.assertFalse( import_utils.search_dump(5, DumpType.FULL, datetime.fromtimestamp(5))) self.assertTrue( import_utils.search_dump(4, DumpType.INCREMENTAL, datetime.fromtimestamp(4))) self.assertFalse( import_utils.search_dump(4, DumpType.INCREMENTAL, datetime.fromtimestamp(5)))