def bootstrap_repo(self, directory): """Bootstrap the repo from Google Stroage if possible. More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing(). """ python_fallback = False if (sys.platform.startswith('win') and not gclient_utils.FindExecutable('7z')): python_fallback = True elif sys.platform.startswith('darwin'): # The OSX version of unzip doesn't support zip64. python_fallback = True elif not gclient_utils.FindExecutable('unzip'): python_fallback = True gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir) gsutil = Gsutil(self.gsutil_exe, boto_path=None) # Get the most recent version of the zipfile. _, ls_out, _ = gsutil.check_call('ls', gs_folder) ls_out_sorted = sorted(ls_out.splitlines()) if not ls_out_sorted: # This repo is not on Google Storage. return False latest_checkout = ls_out_sorted[-1] # Download zip file to a temporary directory. try: tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath()) self.print('Downloading %s' % latest_checkout) code = gsutil.call('cp', latest_checkout, tempdir) if code: return False filename = os.path.join(tempdir, latest_checkout.split('/')[-1]) # Unpack the file with 7z on Windows, unzip on linux, or fallback. if not python_fallback: if sys.platform.startswith('win'): cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename] else: cmd = ['unzip', filename, '-d', directory] retcode = subprocess.call(cmd) else: try: with zipfile.ZipFile(filename, 'r') as f: f.printdir() f.extractall(directory) except Exception as e: self.print('Encountered error: %s' % str(e), file=sys.stderr) retcode = 1 else: retcode = 0 finally: # Clean up the downloaded zipfile. # # This is somehow racy on Windows. # Catching OSError because WindowsError isn't portable and # pylint complains. exponential_backoff_retry( lambda: gclient_utils.rm_file_or_tree(tempdir), excs=(OSError,), name='rmtree [%s]' % (tempdir,), printerr=self.print) if retcode: self.print( 'Extracting bootstrap zipfile %s failed.\n' 'Resuming normal operations.' % filename) return False return True
def bootstrap_repo(self, directory): """Bootstrap the repo from Google Storage if possible. More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing(). """ if not self.bootstrap_bucket: return False python_fallback = ((sys.platform.startswith('win') and not gclient_utils.FindExecutable('7z')) or (not gclient_utils.FindExecutable('unzip')) or ('ZIP64_SUPPORT' not in subprocess.check_output( ["unzip", "-v"]))) gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir) gsutil = Gsutil(self.gsutil_exe, boto_path=None) # Get the most recent version of the zipfile. _, ls_out, ls_err = gsutil.check_call('ls', gs_folder) def compare_filenames(a, b): # |a| and |b| look like gs://.../.../9999.zip. They both have the same # gs://bootstrap_bucket/basedir/ prefix because they come from the same # `gsutil ls`. # This function only compares the numeral parts before .zip. regex_pattern = r'/(\d+)\.zip$' match_a = re.search(regex_pattern, a) match_b = re.search(regex_pattern, b) if (match_a is not None) and (match_b is not None): num_a = int(match_a.group(1)) num_b = int(match_b.group(1)) return cmp(num_a, num_b) # If it doesn't match the format, fallback to string comparison. return cmp(a, b) ls_out_sorted = sorted(ls_out.splitlines(), cmp=compare_filenames) if not ls_out_sorted: # This repo is not on Google Storage. self.print('No bootstrap file for %s found in %s, stderr:\n %s' % (self.mirror_path, self.bootstrap_bucket, ' '.join( (ls_err or '').splitlines(True)))) return False latest_checkout = ls_out_sorted[-1] # Download zip file to a temporary directory. try: tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath()) self.print('Downloading %s' % latest_checkout) with self.print_duration_of('download'): code = gsutil.call('cp', latest_checkout, tempdir) if code: return False filename = os.path.join(tempdir, latest_checkout.split('/')[-1]) # Unpack the file with 7z on Windows, unzip on linux, or fallback. with self.print_duration_of('unzip'): if not python_fallback: if sys.platform.startswith('win'): cmd = [ '7z', 'x', '-o%s' % directory, '-tzip', filename ] else: cmd = ['unzip', filename, '-d', directory] retcode = subprocess.call(cmd) else: try: with zipfile.ZipFile(filename, 'r') as f: f.printdir() f.extractall(directory) except Exception as e: self.print('Encountered error: %s' % str(e), file=sys.stderr) retcode = 1 else: retcode = 0 finally: # Clean up the downloaded zipfile. # # This is somehow racy on Windows. # Catching OSError because WindowsError isn't portable and # pylint complains. exponential_backoff_retry( lambda: gclient_utils.rm_file_or_tree(tempdir), excs=(OSError, ), name='rmtree [%s]' % (tempdir, ), printerr=self.print) if retcode: self.print('Extracting bootstrap zipfile %s failed.\n' 'Resuming normal operations.' % filename) return False return True