def main(args):
parser = optparse.OptionParser(USAGE_STRING)
parser.add_option('-b',
'--bucket',
help='Google Storage bucket to upload to.')
parser.add_option('-e', '--boto', help='Specify a custom boto file.')
parser.add_option('-f',
'--force',
action='store_true',
help='Force upload even if remote file exists.')
parser.add_option('-g',
'--gsutil_path',
default=GSUTIL_DEFAULT_PATH,
help='Path to the gsutil script.')
parser.add_option(
'-m',
'--use_md5',
action='store_true',
help='Generate MD5 files when scanning, and don\'t check '
'the MD5 checksum if a .md5 file is found.')
parser.add_option('-t',
'--num_threads',
default=1,
type='int',
help='Number of uploader threads to run.')
parser.add_option('-s',
'--skip_hashing',
action='store_true',
help='Skip hashing if .sha1 file exists.')
parser.add_option('-0',
'--use_null_terminator',
action='store_true',
help='Use \\0 instead of \\n when parsing '
'the file list from stdin. This is useful if the input '
'is coming from "find ... -print0".')
(options, args) = parser.parse_args()
# Enumerate our inputs.
input_filenames = get_targets(args, parser, options.use_null_terminator)
# Make sure we can find a working instance of gsutil.
if os.path.exists(GSUTIL_DEFAULT_PATH):
gsutil = Gsutil(GSUTIL_DEFAULT_PATH)
else:
gsutil = None
for path in os.environ["PATH"].split(os.pathsep):
if os.path.exists(path) and 'gsutil' in os.listdir(path):
gsutil = Gsutil(os.path.join(path, 'gsutil'))
if not gsutil:
parser.error('gsutil not found in %s, bad depot_tools checkout?' %
GSUTIL_DEFAULT_PATH)
# Check we have a valid bucket with valid permissions.
base_url, code = check_bucket_permissions(options.bucket, gsutil)
if code:
return code
return upload_to_google_storage(input_filenames, base_url, gsutil,
options.force, options.use_md5,
options.num_threads, options.skip_hashing)
def bootstrap_repo(self, directory):
"""Bootstrap the repo from Google Stroage if possible."""
python_fallback = False
if sys.platform.startswith('win') and not self.FindExecutable('7z'):
python_fallback = True
elif sys.platform.startswith('darwin'):
# The OSX version of unzip doesn't support zip64.
python_fallback = True
elif not self.FindExecutable('unzip'):
python_fallback = True
gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
gsutil = Gsutil(
self.gsutil_exe, boto_path=os.devnull, bypass_prodaccess=True)
# Get the most recent version of the zipfile.
_, ls_out, _ = gsutil.check_call('ls', gs_folder)
ls_out_sorted = sorted(ls_out.splitlines())
if not ls_out_sorted:
# This repo is not on Google Storage.
return False
latest_checkout = ls_out_sorted[-1]
# Download zip file to a temporary directory.
try:
tempdir = tempfile.mkdtemp()
self.print('Downloading %s' % latest_checkout)
code, out, err = gsutil.check_call('cp', latest_checkout, tempdir)
if code:
self.print('%s\n%s' % (out, err))
return False
filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
# Unpack the file with 7z on Windows, unzip on linux, or fallback.
if not python_fallback:
if sys.platform.startswith('win'):
cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
else:
cmd = ['unzip', filename, '-d', directory]
retcode = subprocess.call(cmd)
else:
try:
with zipfile.ZipFile(filename, 'r') as f:
f.printdir()
f.extractall(directory)
except Exception as e:
self.print('Encountered error: %s' % str(e), file=sys.stderr)
retcode = 1
else:
retcode = 0
finally:
# Clean up the downloaded zipfile.
gclient_utils.rmtree(tempdir)
if retcode:
self.print(
'Extracting bootstrap zipfile %s failed.\n'
'Resuming normal operations.' % filename)
return False
return True
def bootstrap_repo(self, directory):
"""Bootstrap the repo from Google Stroage if possible.
Requires 7z on Windows and Unzip on Linux/Mac.
"""
if sys.platform.startswith('win'):
if not self.FindExecutable('7z'):
self.print('''
Cannot find 7z in the path. If you want git cache to be able to bootstrap from
Google Storage, please install 7z from:
http://www.7-zip.org/download.html
''')
return False
else:
if not self.FindExecutable('unzip'):
self.print('''
Cannot find unzip in the path. If you want git cache to be able to bootstrap
from Google Storage, please ensure unzip is present on your system.
''')
return False
gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
gsutil = Gsutil(
self.gsutil_exe, boto_path=os.devnull, bypass_prodaccess=True)
# Get the most recent version of the zipfile.
_, ls_out, _ = gsutil.check_call('ls', gs_folder)
ls_out_sorted = sorted(ls_out.splitlines())
if not ls_out_sorted:
# This repo is not on Google Storage.
return False
latest_checkout = ls_out_sorted[-1]
# Download zip file to a temporary directory.
try:
tempdir = tempfile.mkdtemp()
self.print('Downloading %s' % latest_checkout)
code, out, err = gsutil.check_call('cp', latest_checkout, tempdir)
if code:
self.print('%s\n%s' % (out, err))
return False
filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
# Unpack the file with 7z on Windows, or unzip everywhere else.
if sys.platform.startswith('win'):
cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
else:
cmd = ['unzip', filename, '-d', directory]
retcode = subprocess.call(cmd)
finally:
# Clean up the downloaded zipfile.
gclient_utils.rmtree(tempdir)
if retcode:
self.print(
'Extracting bootstrap zipfile %s failed.\n'
'Resuming normal operations.' % filename)
return False
return True
def bootstrap_repo(self, directory):
"""Bootstrap the repo from Google Stroage if possible.
Requires 7z on Windows and Unzip on Linux/Mac.
"""
if sys.platform.startswith('win'):
if not self.FindExecutable('7z'):
self.print('''
Cannot find 7z in the path. If you want git cache to be able to bootstrap from
Google Storage, please install 7z from:
http://www.7-zip.org/download.html
''')
return False
else:
if not self.FindExecutable('unzip'):
self.print('''
Cannot find unzip in the path. If you want git cache to be able to bootstrap
from Google Storage, please ensure unzip is present on your system.
''')
return False
gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
gsutil = Gsutil(self.gsutil_exe,
boto_path=os.devnull,
bypass_prodaccess=True)
# Get the most recent version of the zipfile.
_, ls_out, _ = gsutil.check_call('ls', gs_folder)
ls_out_sorted = sorted(ls_out.splitlines())
if not ls_out_sorted:
# This repo is not on Google Storage.
return False
latest_checkout = ls_out_sorted[-1]
# Download zip file to a temporary directory.
try:
tempdir = tempfile.mkdtemp()
self.print('Downloading %s' % latest_checkout)
code, out, err = gsutil.check_call('cp', latest_checkout, tempdir)
if code:
self.print('%s\n%s' % (out, err))
return False
filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
# Unpack the file with 7z on Windows, or unzip everywhere else.
if sys.platform.startswith('win'):
cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
else:
cmd = ['unzip', filename, '-d', directory]
retcode = subprocess.call(cmd)
finally:
# Clean up the downloaded zipfile.
gclient_utils.rmtree(tempdir)
if retcode:
self.print('Extracting bootstrap zipfile %s failed.\n'
'Resuming normal operations.' % filename)
return False
return True
def bootstrap_repo(self, directory):
"""Bootstrap the repo from Google Stroage if possible.
More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing().
"""
python_fallback = False
if sys.platform.startswith("win") and not gclient_utils.FindExecutable("7z"):
python_fallback = True
elif sys.platform.startswith("darwin"):
# The OSX version of unzip doesn't support zip64.
python_fallback = True
elif not gclient_utils.FindExecutable("unzip"):
python_fallback = True
gs_folder = "gs://%s/%s" % (self.bootstrap_bucket, self.basedir)
gsutil = Gsutil(self.gsutil_exe, boto_path=None)
# Get the most recent version of the zipfile.
_, ls_out, _ = gsutil.check_call("ls", gs_folder)
ls_out_sorted = sorted(ls_out.splitlines())
if not ls_out_sorted:
# This repo is not on Google Storage.
return False
latest_checkout = ls_out_sorted[-1]
# Download zip file to a temporary directory.
try:
tempdir = tempfile.mkdtemp(prefix="_cache_tmp", dir=self.GetCachePath())
self.print("Downloading %s" % latest_checkout)
code = gsutil.call("cp", latest_checkout, tempdir)
if code:
return False
filename = os.path.join(tempdir, latest_checkout.split("/")[-1])
# Unpack the file with 7z on Windows, unzip on linux, or fallback.
if not python_fallback:
if sys.platform.startswith("win"):
cmd = ["7z", "x", "-o%s" % directory, "-tzip", filename]
else:
cmd = ["unzip", filename, "-d", directory]
retcode = subprocess.call(cmd)
else:
try:
with zipfile.ZipFile(filename, "r") as f:
f.printdir()
f.extractall(directory)
except Exception as e:
self.print("Encountered error: %s" % str(e), file=sys.stderr)
retcode = 1
else:
retcode = 0
finally:
# Clean up the downloaded zipfile.
gclient_utils.rm_file_or_tree(tempdir)
if retcode:
self.print("Extracting bootstrap zipfile %s failed.\n" "Resuming normal operations." % filename)
return False
return True
def main():
parser = optparse.OptionParser(USAGE_STRING)
parser.add_option('-b', '--bucket',
help='Google Storage bucket to upload to.')
parser.add_option('-e', '--boto', help='Specify a custom boto file.')
parser.add_option('-a', '--archive', action='store_true',
help='Archive directory as a tar.gz file')
parser.add_option('-f', '--force', action='store_true',
help='Force upload even if remote file exists.')
parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH,
help='Path to the gsutil script.')
parser.add_option('-m', '--use_md5', action='store_true',
help='Generate MD5 files when scanning, and don\'t check '
'the MD5 checksum if a .md5 file is found.')
parser.add_option('-t', '--num_threads', default=1, type='int',
help='Number of uploader threads to run.')
parser.add_option('-s', '--skip_hashing', action='store_true',
help='Skip hashing if .sha1 file exists.')
parser.add_option('-0', '--use_null_terminator', action='store_true',
help='Use \\0 instead of \\n when parsing '
'the file list from stdin. This is useful if the input '
'is coming from "find ... -print0".')
parser.add_option('-z', '--gzip', metavar='ext',
help='Gzip files which end in ext. '
'ext is a comma-separated list')
(options, args) = parser.parse_args()
# Enumerate our inputs.
input_filenames = get_targets(args, parser, options.use_null_terminator)
if options.archive:
if not validate_archive_dirs(input_filenames):
parser.error('Only directories just below cwd are valid entries when '
'using the --archive argument. Entries can not contain .. '
' and entries can not be symlinks. Entries was %s' %
input_filenames)
return 1
input_filenames = create_archives(input_filenames)
# Make sure we can find a working instance of gsutil.
if os.path.exists(GSUTIL_DEFAULT_PATH):
gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
else:
gsutil = None
for path in os.environ["PATH"].split(os.pathsep):
if os.path.exists(path) and 'gsutil' in os.listdir(path):
gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto)
if not gsutil:
parser.error('gsutil not found in %s, bad depot_tools checkout?' %
GSUTIL_DEFAULT_PATH)
base_url = 'gs://%s' % options.bucket
return upload_to_google_storage(
input_filenames, base_url, gsutil, options.force, options.use_md5,
options.num_threads, options.skip_hashing, options.gzip)
def update_bootstrap(self):
# The files are named <git number>.zip
gen_number = subprocess.check_output(
[self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
self.RunGit(['gc']) # Run Garbage Collect to compress packfile.
# Creating a temp file and then deleting it ensures we can use this name.
_, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
os.remove(tmp_zipfile)
subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
dest_name = 'gs://%s/%s/%s.zip' % (
self.bootstrap_bucket, self.basedir, gen_number)
gsutil.call('cp', tmp_zipfile, dest_name)
os.remove(tmp_zipfile)
def update_bootstrap(self):
# The files are named <git number>.zip
gen_number = subprocess.check_output(
[self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
self.RunGit(['gc']) # Run Garbage Collect to compress packfile.
# Creating a temp file and then deleting it ensures we can use this name.
_, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
os.remove(tmp_zipfile)
subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
dest_name = 'gs://%s/%s/%s.zip' % (self.bootstrap_bucket, self.basedir,
gen_number)
gsutil.call('cp', tmp_zipfile, dest_name)
os.remove(tmp_zipfile)
def bootstrap_repo(self, directory):
"""Bootstrap the repo from Google Storage if possible.
More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing().
"""
if not self.bootstrap_bucket:
return False
gsutil = Gsutil(self.gsutil_exe, boto_path=None)
# Get the most recent version of the directory.
# This is determined from the most recent version of a .ready file.
# The .ready file is only uploaded when an entire directory has been
# uploaded to GS.
_, ls_out, ls_err = gsutil.check_call('ls', self._gs_path)
ls_out_set = set(ls_out.strip().splitlines())
latest_dir = self._GetMostRecentCacheDirectory(ls_out_set)
if not latest_dir:
self.print('No bootstrap file for %s found in %s, stderr:\n %s' %
(self.mirror_path, self.bootstrap_bucket, ' '.join(
(ls_err or '').splitlines(True))))
return False
try:
# create new temporary directory locally
tempdir = tempfile.mkdtemp(prefix='_cache_tmp',
dir=self.GetCachePath())
self.RunGit(['init', '--bare'], cwd=tempdir)
self.print('Downloading files in %s/* into %s.' %
(latest_dir, tempdir))
with self.print_duration_of('download'):
code = gsutil.call('-m', 'cp', '-r', latest_dir + "/*",
tempdir)
if code:
return False
# A quick validation that all references are valid.
self.RunGit(['for-each-ref'], cwd=tempdir)
except Exception as e:
self.print('Encountered error: %s' % str(e), file=sys.stderr)
gclient_utils.rmtree(tempdir)
return False
# delete the old directory
if os.path.exists(directory):
gclient_utils.rmtree(directory)
self.Rename(tempdir, directory)
return True
def update_bootstrap(self, prune=False):
# The folder is <git number>
gen_number = subprocess.check_output(
[self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
# Run Garbage Collect to compress packfile.
self.RunGit(['gc', '--prune=all'])
gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
src_name = self.mirror_path
dest_name = '%s/%s' % (self._gs_path, gen_number)
# check to see if folder already exists in gs
_, ls_out, ls_err = gsutil.check_call('ls', dest_name)
_, ls_out_ready, ls_err_ready = (gsutil.check_call(
'ls', dest_name + '.ready'))
# only printing out errors because the folder/ready file
# might not exist yet, so it will error no matter what
if ls_err:
print('Failed to check GS:\n%s' % (ls_err))
if ls_err_ready:
print('Failed to check GS:\n%s' % (ls_err_ready))
if not (ls_out == '' and ls_out_ready == ''):
return
gsutil.call('-m', 'cp', '-r', src_name, dest_name)
#TODO(karenqian): prune old caches
# create .ready file and upload
_, ready_file_name = tempfile.mkstemp(suffix='.ready')
try:
gsutil.call('cp', ready_file_name, '%s.ready' % (dest_name))
finally:
os.remove(ready_file_name)
def update_bootstrap(self, prune=False):
# The files are named <git number>.zip
gen_number = subprocess.check_output(
[self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
self.RunGit(['gc']) # Run Garbage Collect to compress packfile.
# Creating a temp file and then deleting it ensures we can use this name.
_, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
os.remove(tmp_zipfile)
subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
dest_name = '%s/%s.zip' % (gs_folder, gen_number)
gsutil.call('cp', tmp_zipfile, dest_name)
os.remove(tmp_zipfile)
# Remove all other files in the same directory.
if prune:
_, ls_out, _ = gsutil.check_call('ls', gs_folder)
for filename in ls_out.splitlines():
if filename == dest_name:
continue
gsutil.call('rm', filename)
def update_bootstrap(self, prune=False):
# The files are named <git number>.zip
gen_number = subprocess.check_output(
[self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
self.RunGit(['gc']) # Run Garbage Collect to compress packfile.
# Creating a temp file and then deleting it ensures we can use this name.
_, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
os.remove(tmp_zipfile)
subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
dest_name = '%s/%s.zip' % (gs_folder, gen_number)
gsutil.call('cp', tmp_zipfile, dest_name)
os.remove(tmp_zipfile)
# Remove all other files in the same directory.
if prune:
_, ls_out, _ = gsutil.check_call('ls', gs_folder)
for filename in ls_out.splitlines():
if filename == dest_name:
continue
gsutil.call('rm', filename)
def update_bootstrap(self, prune=False):
# The files are named <git number>.zip
gen_number = subprocess.check_output([self.git_exe, "number", "master"], cwd=self.mirror_path).strip()
# Run Garbage Collect to compress packfile.
self.RunGit(["gc", "--prune=all"])
# Creating a temp file and then deleting it ensures we can use this name.
_, tmp_zipfile = tempfile.mkstemp(suffix=".zip")
os.remove(tmp_zipfile)
subprocess.call(["zip", "-r", tmp_zipfile, "."], cwd=self.mirror_path)
gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
gs_folder = "gs://%s/%s" % (self.bootstrap_bucket, self.basedir)
dest_name = "%s/%s.zip" % (gs_folder, gen_number)
gsutil.call("cp", tmp_zipfile, dest_name)
os.remove(tmp_zipfile)
# Remove all other files in the same directory.
if prune:
_, ls_out, _ = gsutil.check_call("ls", gs_folder)
for filename in ls_out.splitlines():
if filename == dest_name:
continue
gsutil.call("rm", filename)
def bootstrap_repo(self, directory):
"""Bootstrap the repo from Google Stroage if possible.
More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing().
"""
python_fallback = False
if (sys.platform.startswith('win') and
not gclient_utils.FindExecutable('7z')):
python_fallback = True
elif sys.platform.startswith('darwin'):
# The OSX version of unzip doesn't support zip64.
python_fallback = True
elif not gclient_utils.FindExecutable('unzip'):
python_fallback = True
gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
gsutil = Gsutil(self.gsutil_exe, boto_path=None)
# Get the most recent version of the zipfile.
_, ls_out, _ = gsutil.check_call('ls', gs_folder)
ls_out_sorted = sorted(ls_out.splitlines())
if not ls_out_sorted:
# This repo is not on Google Storage.
return False
latest_checkout = ls_out_sorted[-1]
# Download zip file to a temporary directory.
try:
tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath())
self.print('Downloading %s' % latest_checkout)
code = gsutil.call('cp', latest_checkout, tempdir)
if code:
return False
filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
# Unpack the file with 7z on Windows, unzip on linux, or fallback.
if not python_fallback:
if sys.platform.startswith('win'):
cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
else:
cmd = ['unzip', filename, '-d', directory]
retcode = subprocess.call(cmd)
else:
try:
with zipfile.ZipFile(filename, 'r') as f:
f.printdir()
f.extractall(directory)
except Exception as e:
self.print('Encountered error: %s' % str(e), file=sys.stderr)
retcode = 1
else:
retcode = 0
finally:
# Clean up the downloaded zipfile.
#
# This is somehow racy on Windows.
# Catching OSError because WindowsError isn't portable and
# pylint complains.
exponential_backoff_retry(
lambda: gclient_utils.rm_file_or_tree(tempdir),
excs=(OSError,),
name='rmtree [%s]' % (tempdir,),
printerr=self.print)
if retcode:
self.print(
'Extracting bootstrap zipfile %s failed.\n'
'Resuming normal operations.' % filename)
return False
return True
def update_bootstrap(self, prune=False, gc_aggressive=False):
# The folder is <git number>
gen_number = subprocess.check_output(
[self.git_exe, 'number', 'master'],
cwd=self.mirror_path).decode('utf-8', 'ignore').strip()
gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
src_name = self.mirror_path
dest_prefix = '%s/%s' % (self._gs_path, gen_number)
# ls_out lists contents in the format: gs://blah/blah/123...
_, ls_out, _ = gsutil.check_call('ls', self._gs_path)
# Check to see if folder already exists in gs
ls_out_set = set(ls_out.strip().splitlines())
if (dest_prefix + '/' in ls_out_set
and dest_prefix + '.ready' in ls_out_set):
print('Cache %s already exists.' % dest_prefix)
return
# Run Garbage Collect to compress packfile.
gc_args = ['gc', '--prune=all']
if gc_aggressive:
gc_args.append('--aggressive')
self.RunGit(gc_args)
gsutil.call('-m', 'cp', '-r', src_name, dest_prefix)
# Create .ready file and upload
_, ready_file_name = tempfile.mkstemp(suffix='.ready')
try:
gsutil.call('cp', ready_file_name, '%s.ready' % (dest_prefix))
finally:
os.remove(ready_file_name)
# remove all other directory/.ready files in the same gs_path
# except for the directory/.ready file previously created
# which can be used for bootstrapping while the current one is
# being uploaded
if not prune:
return
prev_dest_prefix = self._GetMostRecentCacheDirectory(ls_out_set)
if not prev_dest_prefix:
return
for path in ls_out_set:
if (path == prev_dest_prefix + '/'
or path == prev_dest_prefix + '.ready'):
continue
if path.endswith('.ready'):
gsutil.call('rm', path)
continue
gsutil.call('-m', 'rm', '-r', path)
def update_bootstrap(self, prune=False, gc_aggressive=False, branch='master'):
# The folder is <git number>
gen_number = subprocess.check_output(
[self.git_exe, 'number', branch],
cwd=self.mirror_path).decode('utf-8', 'ignore').strip()
gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
src_name = self.mirror_path
dest_prefix = '%s/%s' % (self._gs_path, gen_number)
# ls_out lists contents in the format: gs://blah/blah/123...
_, ls_out, _ = gsutil.check_call('ls', self._gs_path)
# Check to see if folder already exists in gs
ls_out_set = set(ls_out.strip().splitlines())
if (dest_prefix + '/' in ls_out_set and
dest_prefix + '.ready' in ls_out_set):
print('Cache %s already exists.' % dest_prefix)
return
# Reduce the number of individual files to download & write on disk.
self.RunGit(['pack-refs', '--all'])
# Run Garbage Collect to compress packfile.
gc_args = ['gc', '--prune=all']
if gc_aggressive:
# The default "gc --aggressive" is often too aggressive for some machines,
# since it attempts to create as many threads as there are CPU cores,
# while not limiting per-thread memory usage, which puts too much pressure
# on RAM on high-core machines, causing them to thrash. Using lower-level
# commands gives more control over those settings.
# This might not be strictly necessary, but it's fast and is normally run
# by 'gc --aggressive', so it shouldn't hurt.
self.RunGit(['reflog', 'expire', '--all'])
# These are the default repack settings for 'gc --aggressive'.
gc_args = ['repack', '-d', '-l', '-f', '--depth=50', '--window=250', '-A',
'--unpack-unreachable=all']
# A 1G memory limit seems to provide comparable pack results as the
# default, even for our largest repos, while preventing runaway memory (at
# least on current Chromium builders which have about 4G RAM per core).
gc_args.append('--window-memory=1g')
# NOTE: It might also be possible to avoid thrashing with a larger window
# (e.g. "--window-memory=2g") by limiting the number of threads created
# (e.g. "--threads=[cores/2]"). Some limited testing didn't show much
# difference in outcomes on our current repos, but it might be worth
# trying if the repos grow much larger and the packs don't seem to be
# getting compressed enough.
self.RunGit(gc_args)
gsutil.call('-m', 'cp', '-r', src_name, dest_prefix)
# Create .ready file and upload
_, ready_file_name = tempfile.mkstemp(suffix='.ready')
try:
gsutil.call('cp', ready_file_name, '%s.ready' % (dest_prefix))
finally:
os.remove(ready_file_name)
# remove all other directory/.ready files in the same gs_path
# except for the directory/.ready file previously created
# which can be used for bootstrapping while the current one is
# being uploaded
if not prune:
return
prev_dest_prefix = self._GetMostRecentCacheDirectory(ls_out_set)
if not prev_dest_prefix:
return
for path in ls_out_set:
if (path == prev_dest_prefix + '/' or
path == prev_dest_prefix + '.ready'):
continue
if path.endswith('.ready'):
gsutil.call('rm', path)
continue
gsutil.call('-m', 'rm', '-r', path)
def bootstrap_repo(self, directory):
"""Bootstrap the repo from Google Storage if possible.
More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing().
"""
if not self.bootstrap_bucket:
return False
python_fallback = ((sys.platform.startswith('win')
and not gclient_utils.FindExecutable('7z'))
or (not gclient_utils.FindExecutable('unzip'))
or ('ZIP64_SUPPORT' not in subprocess.check_output(
["unzip", "-v"])))
gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
gsutil = Gsutil(self.gsutil_exe, boto_path=None)
# Get the most recent version of the zipfile.
_, ls_out, ls_err = gsutil.check_call('ls', gs_folder)
def compare_filenames(a, b):
# |a| and |b| look like gs://.../.../9999.zip. They both have the same
# gs://bootstrap_bucket/basedir/ prefix because they come from the same
# `gsutil ls`.
# This function only compares the numeral parts before .zip.
regex_pattern = r'/(\d+)\.zip$'
match_a = re.search(regex_pattern, a)
match_b = re.search(regex_pattern, b)
if (match_a is not None) and (match_b is not None):
num_a = int(match_a.group(1))
num_b = int(match_b.group(1))
return cmp(num_a, num_b)
# If it doesn't match the format, fallback to string comparison.
return cmp(a, b)
ls_out_sorted = sorted(ls_out.splitlines(), cmp=compare_filenames)
if not ls_out_sorted:
# This repo is not on Google Storage.
self.print('No bootstrap file for %s found in %s, stderr:\n %s' %
(self.mirror_path, self.bootstrap_bucket, ' '.join(
(ls_err or '').splitlines(True))))
return False
latest_checkout = ls_out_sorted[-1]
# Download zip file to a temporary directory.
try:
tempdir = tempfile.mkdtemp(prefix='_cache_tmp',
dir=self.GetCachePath())
self.print('Downloading %s' % latest_checkout)
with self.print_duration_of('download'):
code = gsutil.call('cp', latest_checkout, tempdir)
if code:
return False
filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
# Unpack the file with 7z on Windows, unzip on linux, or fallback.
with self.print_duration_of('unzip'):
if not python_fallback:
if sys.platform.startswith('win'):
cmd = [
'7z', 'x',
'-o%s' % directory, '-tzip', filename
]
else:
cmd = ['unzip', filename, '-d', directory]
retcode = subprocess.call(cmd)
else:
try:
with zipfile.ZipFile(filename, 'r') as f:
f.printdir()
f.extractall(directory)
except Exception as e:
self.print('Encountered error: %s' % str(e),
file=sys.stderr)
retcode = 1
else:
retcode = 0
finally:
# Clean up the downloaded zipfile.
#
# This is somehow racy on Windows.
# Catching OSError because WindowsError isn't portable and
# pylint complains.
exponential_backoff_retry(
lambda: gclient_utils.rm_file_or_tree(tempdir),
excs=(OSError, ),
name='rmtree [%s]' % (tempdir, ),
printerr=self.print)
if retcode:
self.print('Extracting bootstrap zipfile %s failed.\n'
'Resuming normal operations.' % filename)
return False
return True
def bootstrap_repo(self, directory):
"""Bootstrap the repo from Google Storage if possible.
More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing().
"""
if not self.bootstrap_bucket:
return False
python_fallback = (
(sys.platform.startswith('win') and
not gclient_utils.FindExecutable('7z')) or
(not gclient_utils.FindExecutable('unzip')) or
('ZIP64_SUPPORT' not in subprocess.check_output(["unzip", "-v"]))
)
gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
gsutil = Gsutil(self.gsutil_exe, boto_path=None)
# Get the most recent version of the zipfile.
_, ls_out, ls_err = gsutil.check_call('ls', gs_folder)
ls_out_sorted = sorted(ls_out.splitlines())
if not ls_out_sorted:
# This repo is not on Google Storage.
self.print('No bootstrap file for %s found in %s, stderr:\n %s' %
(self.mirror_path, self.bootstrap_bucket,
' '.join((ls_err or '').splitlines(True))))
return False
latest_checkout = ls_out_sorted[-1]
# Download zip file to a temporary directory.
try:
tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath())
self.print('Downloading %s' % latest_checkout)
with self.print_duration_of('download'):
code = gsutil.call('cp', latest_checkout, tempdir)
if code:
return False
filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
# Unpack the file with 7z on Windows, unzip on linux, or fallback.
with self.print_duration_of('unzip'):
if not python_fallback:
if sys.platform.startswith('win'):
cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
else:
cmd = ['unzip', filename, '-d', directory]
retcode = subprocess.call(cmd)
else:
try:
with zipfile.ZipFile(filename, 'r') as f:
f.printdir()
f.extractall(directory)
except Exception as e:
self.print('Encountered error: %s' % str(e), file=sys.stderr)
retcode = 1
else:
retcode = 0
finally:
# Clean up the downloaded zipfile.
#
# This is somehow racy on Windows.
# Catching OSError because WindowsError isn't portable and
# pylint complains.
exponential_backoff_retry(
lambda: gclient_utils.rm_file_or_tree(tempdir),
excs=(OSError,),
name='rmtree [%s]' % (tempdir,),
printerr=self.print)
if retcode:
self.print(
'Extracting bootstrap zipfile %s failed.\n'
'Resuming normal operations.' % filename)
return False
return True
def bootstrap_repo(self, directory):
"""Bootstrap the repo from Google Stroage if possible."""
python_fallback = False
if sys.platform.startswith('win') and not self.FindExecutable('7z'):
python_fallback = True
elif sys.platform.startswith('darwin'):
# The OSX version of unzip doesn't support zip64.
python_fallback = True
elif not self.FindExecutable('unzip'):
python_fallback = True
gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
gsutil = Gsutil(self.gsutil_exe,
boto_path=os.devnull,
bypass_prodaccess=True)
# Get the most recent version of the zipfile.
_, ls_out, _ = gsutil.check_call('ls', gs_folder)
ls_out_sorted = sorted(ls_out.splitlines())
if not ls_out_sorted:
# This repo is not on Google Storage.
return False
latest_checkout = ls_out_sorted[-1]
# Download zip file to a temporary directory.
try:
tempdir = tempfile.mkdtemp()
self.print('Downloading %s' % latest_checkout)
code, out, err = gsutil.check_call('cp', latest_checkout, tempdir)
if code:
self.print('%s\n%s' % (out, err))
return False
filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
# Unpack the file with 7z on Windows, unzip on linux, or fallback.
if not python_fallback:
if sys.platform.startswith('win'):
cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
else:
cmd = ['unzip', filename, '-d', directory]
retcode = subprocess.call(cmd)
else:
try:
with zipfile.ZipFile(filename, 'r') as f:
f.printdir()
f.extractall(directory)
except Exception as e:
self.print('Encountered error: %s' % str(e),
file=sys.stderr)
retcode = 1
else:
retcode = 0
finally:
# Clean up the downloaded zipfile.
gclient_utils.rmtree(tempdir)
if retcode:
self.print('Extracting bootstrap zipfile %s failed.\n'
'Resuming normal operations.' % filename)
return False
return True
