def import_incremental_dump_to_hdfs(dump_id: int = None) -> str: """ Import the incremental dump with the given dump_id if specified otherwise the latest incremental dump. Notes: All incremental dumps are stored together in incremental.parquet inside the listens directory. Args: dump_id: id of the incremental dump to be imported Returns: the name of the imported dump """ with tempfile.TemporaryDirectory() as temp_dir: src, dump_name, dump_id = ListenbrainzDataDownloader().download_listens( directory=temp_dir, dump_type=DumpType.INCREMENTAL, listens_dump_id=dump_id ) # instantiating ListenbrainzDataUploader creates a spark session which # is a bit non-intuitive. # FIXME in future to make initializing of spark session more explicit? ListenbrainzDataUploader().upload_new_listens_incremental_dump(src) utils.insert_dump_data(dump_id, DumpType.INCREMENTAL, datetime.utcnow()) return dump_name
def test_insert_dump_data(self): """ Test to ensure that data is inserted correctly. """ import_utils.insert_dump_data(9, DumpType.FULL, datetime.fromtimestamp(9)) self.assertTrue( import_utils.search_dump(9, DumpType.FULL, datetime.fromtimestamp(9)))
def import_dump_to_hdfs(dump_type, overwrite, dump_id=None): temp_dir = tempfile.mkdtemp() dump_type = 'incremental' if dump_type == 'incremental' else 'full' src, dump_name, dump_id = ListenbrainzDataDownloader().download_listens( directory=temp_dir, dump_type=dump_type, listens_dump_id=dump_id) ListenbrainzDataUploader().upload_listens(src, overwrite=overwrite) utils.insert_dump_data(dump_id, dump_type, datetime.utcnow()) shutil.rmtree(temp_dir) return dump_name
def test_insert_dump_data_file_missing(self): """ Test to ensure a file is created if it is missing. """ path_found = path_exists(self.path_) if path_found: delete_dir(self.path_, recursive=True) self.assertFalse(import_utils.search_dump(1, "full", datetime.fromtimestamp(1))) import_utils.insert_dump_data(1, "full", datetime.fromtimestamp(1)) self.assertTrue(import_utils.search_dump(1, "full", datetime.fromtimestamp(1)))
def test_insert_dump_data_update_date(self): """ Test to ensure date is updated if entry already exists. """ self.assertFalse( import_utils.search_dump(7, "incremental", datetime.fromtimestamp(9))) import_utils.insert_dump_data(7, "incremental", datetime.fromtimestamp(9)) self.assertTrue( import_utils.search_dump(7, "incremental", datetime.fromtimestamp(9)))
def test_insert_dump_data_update_date(self): """ Test to ensure date is updated if entry already exists. """ self.assertFalse( import_utils.search_dump(7, DumpType.INCREMENTAL, datetime.fromtimestamp(9))) import_utils.insert_dump_data(7, DumpType.INCREMENTAL, datetime.fromtimestamp(9)) self.assertTrue( import_utils.search_dump(7, DumpType.INCREMENTAL, datetime.fromtimestamp(9))) self.assertTrue( import_utils.search_dump(2, DumpType.INCREMENTAL, datetime.fromtimestamp(2)))
def import_full_dump_to_hdfs(dump_id: int = None) -> str: """ Import the full dump with the given dump_id if specified otherwise the latest full dump. Notes: Deletes all the existing listens and uploads listens from new dump. Args: dump_id: id of the full dump to be imported Returns: the name of the imported dump """ with tempfile.TemporaryDirectory() as temp_dir: src, dump_name, dump_id = ListenbrainzDataDownloader( ).download_listens(directory=temp_dir, dump_type=DumpType.FULL, listens_dump_id=dump_id) ListenbrainzDataUploader().upload_new_listens_full_dump(src) utils.insert_dump_data(dump_id, DumpType.FULL, datetime.utcnow()) return dump_name