def CloneToStaging(self, progress_callback=None): """Clones this state to the temporary staging area. This is used for making temporary copies of the entire Cloud SDK installation when doing updates. The entire installation is cloned, but doing so removes any backups and trash from this state before doing the copy. Args: progress_callback: f(float), A function to call with the fraction of completeness. Returns: An InstallationState object for the cloned install. """ self._CreateStateDir() (rm_staging_cb, rm_backup_cb, rm_trash_cb, copy_cb) = (console_io.SplitProgressBar(progress_callback, [1, 1, 1, 7])) self._ClearStaging(progress_callback=rm_staging_cb) self.ClearBackup(progress_callback=rm_backup_cb) self.ClearTrash(progress_callback=rm_trash_cb) class Counter(object): def __init__(self, progress_callback, total): self.count = 0 self.progress_callback = progress_callback self.total = total # This function must match the signature that shutil expects for the # ignore function. def Tick(self, *unused_args): self.count += 1 self.progress_callback(self.count / self.total) return [] if progress_callback: # This takes a little time, so only do it if we are going to report # progress. dirs = set() for _, manifest in six.iteritems(self.InstalledComponents()): dirs.update(manifest.InstalledDirectories()) # There is always the root directory itself and the .install directory. # In general, there could be in the SDK (if people just put stuff in there # but this is fine for an estimate. The progress bar will at worst stay # at 100% for slightly longer. total_dirs = len(dirs) + 2 ticker = Counter(copy_cb, total_dirs).Tick if total_dirs else None else: ticker = None shutil.copytree(self.__sdk_root, self.__sdk_staging_root, symlinks=True, ignore=ticker) staging_state = InstallationState(self.__sdk_staging_root) # pylint: disable=protected-access, This is an instance of InstallationState staging_state._CreateStateDir() return staging_state
def Start(self, total_files): self._progress_bar = console_io.ProgressBar( 'Uploading {0} file(s)'.format(total_files)) (self.read_progress, self.write_progress) = (console_io.SplitProgressBar( self._progress_bar.SetProgress, [1, 6])) self._workspace.SetPostCallback(self._UpdateWriteProgress) self.total_files = total_files self._progress_bar.Start()
def testDefault(self): with console_io.ProgressBar('', total_ticks=10): callbacks = console_io.SplitProgressBar(None, [.1, .4, .5]) self.assertEqual(3, len(callbacks)) self.AssertErrEquals('1==========2\n' '3= =4\n' '5') callbacks[0](1) self.AssertErrEquals('1==========2\n' '3= =4\n' '5') callbacks[1](1) self.AssertErrEquals('1==========2\n' '3= =4\n' '5') callbacks[2](1) self.AssertErrEquals('1==========2\n' '3= =4\n' '5')
def testSplit(self): with console_io.ProgressBar('', total_ticks=10) as pb: callbacks = console_io.SplitProgressBar(pb.SetProgress, [.1, .4, .5]) self.assertEqual(3, len(callbacks)) self.AssertErrEquals('1==========2\n' '3= =4\n' '5') callbacks[0](.5) self.AssertErrEquals('1==========2\n' '3= =4\n' '5') callbacks[0](1) self.AssertErrEquals('1==========2\n' '3= =4\n' '5=') callbacks[1](.5) self.AssertErrEquals('1==========2\n' '3= =4\n' '5===') callbacks[1](1) self.AssertErrEquals('1==========2\n' '3= =4\n' '5=====') callbacks[2](.5) self.AssertErrEquals('1==========2\n' '3= =4\n' '5=======') callbacks[2](1) self.AssertErrEquals('1==========2\n' '3= =4\n' '5==========6\n')
def DownloadAndExtractTar(url, download_dir, extract_dir, progress_callback=None, command_path='unknown'): """Download and extract the given tar file. Args: url: str, The URL to download. download_dir: str, The path to put the temporary download file into. extract_dir: str, The path to extract the tar into. progress_callback: f(float), A function to call with the fraction of completeness. command_path: the command path to include in the User-Agent header if the URL is HTTP Returns: [str], The files that were extracted from the tar file. Raises: URLFetchError: If there is a problem fetching the given URL. """ for d in [download_dir, extract_dir]: if not os.path.exists(d): file_utils.MakeDir(d) download_file_path = os.path.join(download_dir, os.path.basename(url)) if os.path.exists(download_file_path): os.remove(download_file_path) (download_callback, install_callback) = ( console_io.SplitProgressBar(progress_callback, [1, 1])) try: req = ComponentInstaller.MakeRequest(url, command_path) try: total_size = float(req.info().get('Content-length', '0')) # pylint: disable=broad-except, We never want progress bars to block an # update. except Exception: total_size = 0 with file_utils.BinaryFileWriter(download_file_path) as fp: # This is the buffer size that shutil.copyfileobj uses. buf_size = 16*1024 total_written = 0 while True: buf = req.read(buf_size) if not buf: break fp.write(buf) total_written += len(buf) if total_size: download_callback(total_written / total_size) download_callback(1) except (urllib.error.HTTPError, urllib.error.URLError, ssl.SSLError) as e: raise URLFetchError(e) with tarfile.open(name=download_file_path) as tar: members = tar.getmembers() total_files = len(members) files = [] for num, member in enumerate(members, start=1): files.append(member.name + '/' if member.isdir() else member.name) tar.extract(member, extract_dir) install_callback(num / total_files) install_callback(1) os.remove(download_file_path) return files
def DownloadAndExtractTar(url, download_dir, extract_dir, progress_callback=None, command_path='unknown'): """Download and extract the given tar file. Args: url: str, The URL to download. download_dir: str, The path to put the temporary download file into. extract_dir: str, The path to extract the tar into. progress_callback: f(float), A function to call with the fraction of completeness. command_path: the command path to include in the User-Agent header if the URL is HTTP Returns: [str], The files that were extracted from the tar file. Raises: URLFetchError: If there is a problem fetching the given URL. """ for d in [download_dir, extract_dir]: if not os.path.exists(d): file_utils.MakeDir(d) download_file_path = os.path.join(download_dir, os.path.basename(url)) if os.path.exists(download_file_path): os.remove(download_file_path) (download_callback, install_callback) = (console_io.SplitProgressBar(progress_callback, [1, 1])) try: response = MakeRequest(url, command_path) with file_utils.BinaryFileWriter(download_file_path) as fp: total_written = 0 total_size = len(response.content) for chunk in response.iter_content(chunk_size=WRITE_BUFFER_SIZE): fp.write(chunk) total_written += len(chunk) download_callback(total_written / total_size) download_callback(1) except (requests.exceptions.HTTPError, OSError) as e: raise URLFetchError(e) with tarfile.open(name=download_file_path) as tar: members = tar.getmembers() total_files = len(members) files = [] for num, member in enumerate(members, start=1): files.append(member.name + '/' if member.isdir() else member.name) tar.extract(member, extract_dir) full_path = os.path.join(extract_dir, member.name) # Ensure read-and-write permission for all files if os.path.isfile(full_path) and not os.access(full_path, os.W_OK): os.chmod(full_path, stat.S_IWUSR | stat.S_IREAD) install_callback(num / total_files) install_callback(1) os.remove(download_file_path) return files