def upload_release_json_dump(self, archive: str): """ Decompress archive and upload artist relation to HDFS. Args: archive: artist relation tar file to upload. """ hdfs_dir = path.MUSICBRAINZ_RELEASE_DUMP hdfs_mbdump_dir = os.path.join( hdfs_dir, "mbdump" ) # release.tar.xz file has actual dump file inside mbdump dir with tarfile.open(name=archive, mode="r:xz" ) as tar, tempfile.TemporaryDirectory() as local_dir: # Remove existing dumps if utils.path_exists(hdfs_dir): utils.delete_dir(hdfs_dir, recursive=True) utils.create_dir(hdfs_dir) for member in tar: t0 = time.monotonic() logger.info(f"Extracting {member.name}") tar.extract(member, path=local_dir) logger.info( f"Done. Total time: {time.monotonic() - t0:.2f} sec") t0 = time.monotonic() logger.info(f"Uploading {member.name}") hdfs_path = os.path.join(hdfs_dir, member.name) local_path = os.path.join(local_dir, member.name) utils.upload_to_HDFS(hdfs_path, local_path) logger.info( f"Done. Total time: {time.monotonic() - t0:.2f} sec")
def setUpClass(cls) -> None: super(RecommendationsTestCase, cls).setUpClass() utils.upload_to_HDFS( LISTENBRAINZ_NEW_DATA_DIRECTORY, os.path.join(TEST_DATA_PATH, 'rec_listens.parquet')) utils.upload_to_HDFS( RECOMMENDATION_RECORDING_MAPPED_LISTENS, os.path.join(TEST_DATA_PATH, 'mapped_listens.parquet'))
def test_upload_to_HDFS(self): temp_file = tempfile.mkdtemp() local_path = os.path.join(temp_file, 'test_file.txt') with open(local_path, 'w') as f: f.write('test file') self.path_ = '/test/upload.parquet' utils.upload_to_HDFS(self.path_, local_path) status = utils.path_exists(self.path_) self.assertTrue(status)
def extract_and_upload_archive(self, archive, local_dir, hdfs_dir, cleanup_on_failure=True): """ Extract the archive and upload it to the given hdfs directory. Args: archive: path to the tar archive to uploaded local_dir: path to local dir to be used for extraction hdfs_dir: path to hdfs dir where contents of tar should be uploaded cleanup_on_failure: whether to delete local and hdfs directories if error occurs during extraction """ total_files = 0 total_time = 0.0 with tarfile.open(archive, mode='r') as tar: for member in tar: if member.isfile() and member.name.endswith(".parquet"): logger.info(f"Uploading {member.name}...") t0 = time.monotonic() try: tar.extract(member, path=local_dir) except tarfile.TarError as err: if cleanup_on_failure: if utils.path_exists(hdfs_dir): utils.delete_dir(hdfs_dir, recursive=True) shutil.rmtree(local_dir, ignore_errors=True) raise DumpInvalidException( f"{type(err).__name__} while extracting {member.name}, aborting import" ) hdfs_path = os.path.join(hdfs_dir, member.name) local_path = os.path.join(local_dir, member.name) utils.upload_to_HDFS(hdfs_path, local_path) time_taken = time.monotonic() - t0 total_files += 1 total_time += time_taken logger.info( f"Done! Current file processed in {time_taken:.2f} sec" ) logger.info( f"Done! Total files processed {total_files}. Average time taken: {total_time / total_files:.2f}" )
def upload_archive(self, tmp_dump_dir, tar, dest_path, schema, callback=None, force=False): """ Upload data dump to HDFS. Args: tmp_dump_dir (str): Path to temporary directory to upload JSON. tar: Uncompressed tar object. dest_path (str): HDFS path to upload data dump. schema: Schema of parquet to be uploaded. callback: Function to process JSON files. force: If True deletes dir at dest_path """ if callback is None: raise NotImplementedError('Callback to process JSON missing. Aboritng...') if force: current_app.logger.info('Removing {} from HDFS...'.format(dest_path)) utils.delete_dir(dest_path, recursive=True) current_app.logger.info('Done!') file_count = 0 total_time = 0.0 for member in tar: if member.isfile() and self._is_json_file(member.name): current_app.logger.info('Loading {}...'.format(member.name)) t = time.time() tar.extract(member) tmp_hdfs_path = os.path.join(tmp_dump_dir, member.name) utils.upload_to_HDFS(tmp_hdfs_path, member.name) callback(member.name, dest_path, tmp_hdfs_path, schema) utils.delete_dir(tmp_hdfs_path, recursive=True) os.remove(member.name) file_count += 1 time_taken = time.time() - t current_app.logger.info("Done! Processed {} files. Current file done in {:.2f} sec".format( file_count, time_taken)) total_time += time_taken average_time = total_time / file_count current_app.logger.info("Total time: {:.2f}, average time: {:.2f}".format(total_time, average_time)) utils.delete_dir(tmp_dump_dir, recursive=True) shutil.rmtree(tmp_dump_dir)
def upload_archive(self, tmp_dump_dir, tar, dest_path, schema, callback=None, overwrite=False): """ Upload data dump to HDFS. Args: tmp_dump_dir (str): Path to temporary directory to upload JSON. tar: Uncompressed tar object. dest_path (str): HDFS path to upload data dump. schema: Schema of parquet to be uploaded. callback: Function to process JSON files. overwrite: If True deletes dir at dest_path """ if callback is None: raise NotImplementedError( 'Callback to process JSON missing. Aborting...') # Delete TEMP_DIR_PATH if it exists if utils.path_exists(TEMP_DIR_PATH): utils.delete_dir(TEMP_DIR_PATH, recursive=True) # Copy data from dest_path to TEMP_DIR_PATH to be merged with new data if not overwrite and utils.path_exists(dest_path): t0 = time.monotonic() logger.info("Copying old listens into '{}'".format(TEMP_DIR_PATH)) utils.copy(dest_path, TEMP_DIR_PATH, overwrite=True) logger.info("Done! Time taken: {:.2f}".format(time.monotonic() - t0)) logger.info("Uploading listens to temporary directory in HDFS...") total_files = 0 total_time = 0.0 for member in tar: if member.isfile() and self._is_json_file(member.name): logger.info("Uploading {}...".format(member.name)) t0 = time.monotonic() try: tar.extract(member) except TarError as err: # Cleanup if utils.path_exists(TEMP_DIR_PATH): utils.delete_dir(TEMP_DIR_PATH, recursive=True) if utils.path_exists(tmp_dump_dir): utils.delete_dir(tmp_dump_dir, recursive=True) raise DumpInvalidException( "{} while extracting {}, aborting import".format( type(err).__name__, member.name)) tmp_hdfs_path = os.path.join(tmp_dump_dir, member.name) utils.upload_to_HDFS(tmp_hdfs_path, member.name) callback(member.name, TEMP_DIR_PATH, tmp_hdfs_path, not overwrite, schema) utils.delete_dir(tmp_hdfs_path, recursive=True) os.remove(member.name) time_taken = time.monotonic() - t0 total_files += 1 total_time += time_taken logger.info( "Done! Current file processed in {:.2f} sec".format( time_taken)) logger.info( "Done! Total files processed {}. Average time taken: {:.2f}". format(total_files, total_time / total_files)) # Delete dest_path if present if utils.path_exists(dest_path): logger.info('Removing {} from HDFS...'.format(dest_path)) utils.delete_dir(dest_path, recursive=True) logger.info('Done!') logger.info("Moving the processed files to {}".format(dest_path)) t0 = time.monotonic() # Check if parent directory exists, if not create a directory dest_path_parent = pathlib.Path(dest_path).parent if not utils.path_exists(dest_path_parent): utils.create_dir(dest_path_parent) utils.rename(TEMP_DIR_PATH, dest_path) utils.logger.info("Done! Time taken: {:.2f}".format(time.monotonic() - t0)) # Cleanup utils.delete_dir(tmp_dump_dir, recursive=True)