def CloneToStaging(self, progress_callback=None):
"""Clones this state to the temporary staging area.
This is used for making temporary copies of the entire Cloud SDK
installation when doing updates. The entire installation is cloned, but
doing so removes any backups and trash from this state before doing the
copy.
Args:
progress_callback: f(float), A function to call with the fraction of
completeness.
Returns:
An InstallationState object for the cloned install.
"""
self._CreateStateDir()
(rm_staging_cb, rm_backup_cb, rm_trash_cb,
copy_cb) = (console_io.SplitProgressBar(progress_callback,
[1, 1, 1, 7]))
self._ClearStaging(progress_callback=rm_staging_cb)
self.ClearBackup(progress_callback=rm_backup_cb)
self.ClearTrash(progress_callback=rm_trash_cb)
class Counter(object):
def __init__(self, progress_callback, total):
self.count = 0
self.progress_callback = progress_callback
self.total = total
# This function must match the signature that shutil expects for the
# ignore function.
def Tick(self, *unused_args):
self.count += 1
self.progress_callback(self.count / self.total)
return []
if progress_callback:
# This takes a little time, so only do it if we are going to report
# progress.
dirs = set()
for _, manifest in six.iteritems(self.InstalledComponents()):
dirs.update(manifest.InstalledDirectories())
# There is always the root directory itself and the .install directory.
# In general, there could be in the SDK (if people just put stuff in there
# but this is fine for an estimate. The progress bar will at worst stay
# at 100% for slightly longer.
total_dirs = len(dirs) + 2
ticker = Counter(copy_cb, total_dirs).Tick if total_dirs else None
else:
ticker = None
shutil.copytree(self.__sdk_root,
self.__sdk_staging_root,
symlinks=True,
ignore=ticker)
staging_state = InstallationState(self.__sdk_staging_root)
# pylint: disable=protected-access, This is an instance of InstallationState
staging_state._CreateStateDir()
return staging_state
def Start(self, total_files):
self._progress_bar = console_io.ProgressBar(
'Uploading {0} file(s)'.format(total_files))
(self.read_progress,
self.write_progress) = (console_io.SplitProgressBar(
self._progress_bar.SetProgress, [1, 6]))
self._workspace.SetPostCallback(self._UpdateWriteProgress)
self.total_files = total_files
self._progress_bar.Start()
def testDefault(self):
with console_io.ProgressBar('', total_ticks=10):
callbacks = console_io.SplitProgressBar(None, [.1, .4, .5])
self.assertEqual(3, len(callbacks))
self.AssertErrEquals('1==========2\n' '3= =4\n' '5')
callbacks[0](1)
self.AssertErrEquals('1==========2\n' '3= =4\n' '5')
callbacks[1](1)
self.AssertErrEquals('1==========2\n' '3= =4\n' '5')
callbacks[2](1)
self.AssertErrEquals('1==========2\n' '3= =4\n' '5')
def testSplit(self):
with console_io.ProgressBar('', total_ticks=10) as pb:
callbacks = console_io.SplitProgressBar(pb.SetProgress,
[.1, .4, .5])
self.assertEqual(3, len(callbacks))
self.AssertErrEquals('1==========2\n' '3= =4\n' '5')
callbacks[0](.5)
self.AssertErrEquals('1==========2\n' '3= =4\n' '5')
callbacks[0](1)
self.AssertErrEquals('1==========2\n' '3= =4\n' '5=')
callbacks[1](.5)
self.AssertErrEquals('1==========2\n' '3= =4\n' '5===')
callbacks[1](1)
self.AssertErrEquals('1==========2\n' '3= =4\n' '5=====')
callbacks[2](.5)
self.AssertErrEquals('1==========2\n' '3= =4\n' '5=======')
callbacks[2](1)
self.AssertErrEquals('1==========2\n'
'3= =4\n'
'5==========6\n')
def DownloadAndExtractTar(url, download_dir, extract_dir,
progress_callback=None, command_path='unknown'):
"""Download and extract the given tar file.
Args:
url: str, The URL to download.
download_dir: str, The path to put the temporary download file into.
extract_dir: str, The path to extract the tar into.
progress_callback: f(float), A function to call with the fraction of
completeness.
command_path: the command path to include in the User-Agent header if the
URL is HTTP
Returns:
[str], The files that were extracted from the tar file.
Raises:
URLFetchError: If there is a problem fetching the given URL.
"""
for d in [download_dir, extract_dir]:
if not os.path.exists(d):
file_utils.MakeDir(d)
download_file_path = os.path.join(download_dir, os.path.basename(url))
if os.path.exists(download_file_path):
os.remove(download_file_path)
(download_callback, install_callback) = (
console_io.SplitProgressBar(progress_callback, [1, 1]))
try:
req = ComponentInstaller.MakeRequest(url, command_path)
try:
total_size = float(req.info().get('Content-length', '0'))
# pylint: disable=broad-except, We never want progress bars to block an
# update.
except Exception:
total_size = 0
with file_utils.BinaryFileWriter(download_file_path) as fp:
# This is the buffer size that shutil.copyfileobj uses.
buf_size = 16*1024
total_written = 0
while True:
buf = req.read(buf_size)
if not buf:
break
fp.write(buf)
total_written += len(buf)
if total_size:
download_callback(total_written / total_size)
download_callback(1)
except (urllib.error.HTTPError,
urllib.error.URLError,
ssl.SSLError) as e:
raise URLFetchError(e)
with tarfile.open(name=download_file_path) as tar:
members = tar.getmembers()
total_files = len(members)
files = []
for num, member in enumerate(members, start=1):
files.append(member.name + '/' if member.isdir() else member.name)
tar.extract(member, extract_dir)
install_callback(num / total_files)
install_callback(1)
os.remove(download_file_path)
return files
def DownloadAndExtractTar(url,
download_dir,
extract_dir,
progress_callback=None,
command_path='unknown'):
"""Download and extract the given tar file.
Args:
url: str, The URL to download.
download_dir: str, The path to put the temporary download file into.
extract_dir: str, The path to extract the tar into.
progress_callback: f(float), A function to call with the fraction of
completeness.
command_path: the command path to include in the User-Agent header if the
URL is HTTP
Returns:
[str], The files that were extracted from the tar file.
Raises:
URLFetchError: If there is a problem fetching the given URL.
"""
for d in [download_dir, extract_dir]:
if not os.path.exists(d):
file_utils.MakeDir(d)
download_file_path = os.path.join(download_dir, os.path.basename(url))
if os.path.exists(download_file_path):
os.remove(download_file_path)
(download_callback,
install_callback) = (console_io.SplitProgressBar(progress_callback,
[1, 1]))
try:
response = MakeRequest(url, command_path)
with file_utils.BinaryFileWriter(download_file_path) as fp:
total_written = 0
total_size = len(response.content)
for chunk in response.iter_content(chunk_size=WRITE_BUFFER_SIZE):
fp.write(chunk)
total_written += len(chunk)
download_callback(total_written / total_size)
download_callback(1)
except (requests.exceptions.HTTPError, OSError) as e:
raise URLFetchError(e)
with tarfile.open(name=download_file_path) as tar:
members = tar.getmembers()
total_files = len(members)
files = []
for num, member in enumerate(members, start=1):
files.append(member.name + '/' if member.isdir() else member.name)
tar.extract(member, extract_dir)
full_path = os.path.join(extract_dir, member.name)
# Ensure read-and-write permission for all files
if os.path.isfile(full_path) and not os.access(full_path, os.W_OK):
os.chmod(full_path, stat.S_IWUSR | stat.S_IREAD)
install_callback(num / total_files)
install_callback(1)
os.remove(download_file_path)
return files
