def download_and_uncompress(self, fileobj, dst_path): """Streams the content for the 'fileobj' and stores the result in dst_path. Args: fileobj: File handle pointing to .tar/.tar.gz content. dst_path: Absolute path where to store uncompressed data from 'fileobj'. Raises: ValueError: Unknown object encountered inside the TAR file. """ try: with tarfile.open(mode="r|*", fileobj=fileobj) as tgz: for tarinfo in tgz: abs_target_path = _merge_relative_path( dst_path, tarinfo.name) if tarinfo.isfile(): self._extract_file(tgz, tarinfo, abs_target_path) elif tarinfo.isdir(): tf_v1.gfile.MakeDirs(abs_target_path) else: # We do not support symlinks and other uncommon objects. raise ValueError( "Unexpected object type in tar archive: %s" % tarinfo.type) total_size_str = tf_utils.bytes_to_readable_str( self._total_bytes_downloaded, True) self._print_download_progress_msg( "Downloaded %s, Total size: %s" % (self._url, total_size_str), flush=True) except tarfile.ReadError: raise IOError("%s does not appear to be a valid module." % self._url)
def _log_progress(self, bytes_downloaded): """Logs progress information about ongoing module download. Args: bytes_downloaded: Number of bytes downloaded. """ self._total_bytes_downloaded += bytes_downloaded now = time.time() if (self._interactive_mode() or now - self._last_progress_msg_print_time > 15): # Print progress message every 15 secs or if interactive progress # tracking is enabled. self._print_download_progress_msg( "Downloading %s: %s" % (self._url, tf_utils.bytes_to_readable_str( self._total_bytes_downloaded, True))) self._last_progress_msg_print_time = now
def _log_progress(self, bytes_downloaded): """Logs progress information about ongoing module download. Args: bytes_downloaded: Number of bytes downloaded. """ self._total_bytes_downloaded += bytes_downloaded now = time.time() if (self._interactive_mode() or now - self._last_progress_msg_print_time > 15): # Print progress message every 15 secs or if interactive progress # tracking is enabled. self._print_download_progress_msg( "Downloading %s: %s" % (self._url, tf_utils.bytes_to_readable_str( self._total_bytes_downloaded, True))) self._last_progress_msg_print_time = now
def download_and_uncompress(self, fileobj, dst_path): """Streams the content for the 'fileobj' and stores the result in dst_path. Args: fileobj: File handle pointing to .tar/.tar.gz content. dst_path: Absolute path where to store uncompressed data from 'fileobj'. Raises: ValueError: Unknown object encountered inside the TAR file. """ try: file_utils.extract_tarfile_to_destination( fileobj, dst_path, log_function=self._log_progress) total_size_str = tf_utils.bytes_to_readable_str( self._total_bytes_downloaded, True) self._print_download_progress_msg("Downloaded %s, Total size: %s" % (self._url, total_size_str), flush=True) except tarfile.ReadError: raise IOError("%s does not appear to be a valid module." % self._url)
def download_and_uncompress(self, fileobj, dst_path): """Streams the content for the 'fileobj' and stores the result in dst_path. Args: fileobj: File handle pointing to .tar/.tar.gz content. dst_path: Absolute path where to store uncompressed data from 'fileobj'. Raises: ValueError: Unknown object encountered inside the TAR file. """ try: with tarfile.open(mode="r|*", fileobj=fileobj) as tgz: for tarinfo in tgz: if tarinfo.name.startswith("/"): tarinfo.name = tarinfo.name[1:] # Check that the absolute path of the object to extract is inside # `path`. abs_target_path = os.path.join(dst_path, tarinfo.name) if not abs_target_path.startswith(dst_path): raise ValueError( "Module archive contains files outside its directory") if tarinfo.isfile(): self._extract_file(tgz, tarinfo, abs_target_path) elif tarinfo.isdir(): tf.gfile.MakeDirs(abs_target_path) else: # We do not support symlinks and other uncommon objects. raise ValueError( "Unexpected object type in tar archive: %s" % tarinfo.type) total_size_str = tf_utils.bytes_to_readable_str( self._total_bytes_downloaded, True) self._print_download_progress_msg( "Downloaded %s, Total size: %s" % (self._url, total_size_str), flush=True) except tarfile.ReadError: raise IOError("%s does not appear to be a valid module." % self._url)