def test_search_dump(self):
""" Test to ensure 'True' is returned if appropriate dump is found and 'False' if it isn't found. """
self.assertTrue(import_utils.search_dump(4, "full", datetime.fromtimestamp(3)))
self.assertFalse(import_utils.search_dump(4, "full", datetime.fromtimestamp(5)))
self.assertFalse(import_utils.search_dump(5, "full", datetime.fromtimestamp(5)))
self.assertTrue(import_utils.search_dump(4, "incremental", datetime.fromtimestamp(4)))
self.assertFalse(import_utils.search_dump(4, "incremental", datetime.fromtimestamp(5)))
def test_insert_dump_data_file_missing(self):
""" Test to ensure a file is created if it is missing. """
path_found = path_exists(self.path_)
if path_found:
delete_dir(self.path_, recursive=True)
self.assertFalse(import_utils.search_dump(1, "full", datetime.fromtimestamp(1)))
import_utils.insert_dump_data(1, "full", datetime.fromtimestamp(1))
self.assertTrue(import_utils.search_dump(1, "full", datetime.fromtimestamp(1)))
def test_insert_dump_data_update_date(self):
""" Test to ensure date is updated if entry already exists. """
self.assertFalse(
import_utils.search_dump(7, "incremental",
datetime.fromtimestamp(9)))
import_utils.insert_dump_data(7, "incremental",
datetime.fromtimestamp(9))
self.assertTrue(
import_utils.search_dump(7, "incremental",
datetime.fromtimestamp(9)))
def test_insert_dump_data_update_date(self):
""" Test to ensure date is updated if entry already exists. """
self.assertFalse(
import_utils.search_dump(7, DumpType.INCREMENTAL,
datetime.fromtimestamp(9)))
import_utils.insert_dump_data(7, DumpType.INCREMENTAL,
datetime.fromtimestamp(9))
self.assertTrue(
import_utils.search_dump(7, DumpType.INCREMENTAL,
datetime.fromtimestamp(9)))
self.assertTrue(
import_utils.search_dump(2, DumpType.INCREMENTAL,
datetime.fromtimestamp(2)))
def import_newest_incremental_dump_handler():
imported_dumps = []
latest_full_dump = utils.get_latest_full_dump()
if latest_full_dump is None:
# If no prior full dump is present, just import the lates incremental dump
imported_dumps.append(
import_dump_to_hdfs('incremental', overwrite=False))
current_app.logger.warn(
"No previous full dump found, importing latest incremental dump",
exc_info=True)
else:
# Import all missing dumps from last full dump import
dump_id = latest_full_dump["dump_id"] + 1
imported_at = latest_full_dump["imported_at"]
while True:
if not utils.search_dump(dump_id, 'incremental', imported_at):
try:
imported_dumps.append(
import_dump_to_hdfs('incremental', False, dump_id))
except DumpNotFoundException:
break
except Exception as e:
# Exit if any other error occurs during import
current_app.logger.error(
f"Error while importing incremental dump with ID {dump_id}: {e}",
exc_info=True)
break
dump_id += 1
return [{
'type': 'import_incremental_dump',
'imported_dump': imported_dumps,
'time': str(datetime.utcnow()),
}]
def test_search_dump_file_missing(self):
""" Test to ensure 'False' is returned if metadata file is missing. """
path_found = path_exists(self.path_)
if path_found:
delete_dir(self.path_, recursive=True)
self.assertFalse(import_utils.search_dump(1, "full", datetime.fromtimestamp(1)))
def import_newest_incremental_dump_handler():
errors = []
imported_dumps = []
latest_full_dump = utils.get_latest_full_dump()
if latest_full_dump is None:
# If no prior full dump is present, just import the latest incremental dump
imported_dumps.append(import_incremental_dump_to_hdfs(dump_id=None))
error_msg = "No previous full dump found, importing latest incremental dump"
errors.append(error_msg)
logger.warning(error_msg, exc_info=True)
else:
# Import all missing dumps from last full dump import
start_id = latest_full_dump["dump_id"] + 1
imported_at = latest_full_dump["imported_at"]
end_id = ListenbrainzDataDownloader().get_latest_dump_id(DumpType.INCREMENTAL) + 1
for dump_id in range(start_id, end_id, 1):
if not utils.search_dump(dump_id, DumpType.INCREMENTAL, imported_at):
try:
imported_dumps.append(import_incremental_dump_to_hdfs(dump_id))
except Exception as e:
# Skip current dump if any error occurs during import
error_msg = f"Error while importing incremental dump with ID {dump_id}: {e}"
errors.append(error_msg)
logger.error(error_msg, exc_info=True)
continue
dump_id += 1
request_consumer.rc.ping()
return [{
'type': 'import_incremental_dump',
'imported_dump': imported_dumps,
'errors': errors,
'time': str(datetime.utcnow()),
}]
def test_insert_dump_data(self):
""" Test to ensure that data is inserted correctly. """
import_utils.insert_dump_data(9, DumpType.FULL,
datetime.fromtimestamp(9))
self.assertTrue(
import_utils.search_dump(9, DumpType.FULL,
datetime.fromtimestamp(9)))
def test_search_dump(self):
""" Test to ensure 'True' is returned if appropriate dump is found and 'False' if it isn't found. """
self.assertTrue(
import_utils.search_dump(4, DumpType.FULL,
datetime.fromtimestamp(3)))
self.assertFalse(
import_utils.search_dump(4, DumpType.FULL,
datetime.fromtimestamp(5)))
self.assertFalse(
import_utils.search_dump(5, DumpType.FULL,
datetime.fromtimestamp(5)))
self.assertTrue(
import_utils.search_dump(4, DumpType.INCREMENTAL,
datetime.fromtimestamp(4)))
self.assertFalse(
import_utils.search_dump(4, DumpType.INCREMENTAL,
datetime.fromtimestamp(5)))
