def bootstrap_repo(self, directory): """Bootstrap the repo from Google Stroage if possible. More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing(). """ python_fallback = False if sys.platform.startswith("win") and not gclient_utils.FindExecutable("7z"): python_fallback = True elif sys.platform.startswith("darwin"): # The OSX version of unzip doesn't support zip64. python_fallback = True elif not gclient_utils.FindExecutable("unzip"): python_fallback = True gs_folder = "gs://%s/%s" % (self.bootstrap_bucket, self.basedir) gsutil = Gsutil(self.gsutil_exe, boto_path=None) # Get the most recent version of the zipfile. _, ls_out, _ = gsutil.check_call("ls", gs_folder) ls_out_sorted = sorted(ls_out.splitlines()) if not ls_out_sorted: # This repo is not on Google Storage. return False latest_checkout = ls_out_sorted[-1] # Download zip file to a temporary directory. try: tempdir = tempfile.mkdtemp(prefix="_cache_tmp", dir=self.GetCachePath()) self.print("Downloading %s" % latest_checkout) code = gsutil.call("cp", latest_checkout, tempdir) if code: return False filename = os.path.join(tempdir, latest_checkout.split("/")[-1]) # Unpack the file with 7z on Windows, unzip on linux, or fallback. if not python_fallback: if sys.platform.startswith("win"): cmd = ["7z", "x", "-o%s" % directory, "-tzip", filename] else: cmd = ["unzip", filename, "-d", directory] retcode = subprocess.call(cmd) else: try: with zipfile.ZipFile(filename, "r") as f: f.printdir() f.extractall(directory) except Exception as e: self.print("Encountered error: %s" % str(e), file=sys.stderr) retcode = 1 else: retcode = 0 finally: # Clean up the downloaded zipfile. gclient_utils.rm_file_or_tree(tempdir) if retcode: self.print("Extracting bootstrap zipfile %s failed.\n" "Resuming normal operations." % filename) return False return True
def UnlockAll(cls): cachepath = cls.GetCachePath() if not cachepath: return dirlist = os.listdir(cachepath) repo_dirs = set([os.path.join(cachepath, path) for path in dirlist if os.path.isdir(os.path.join(cachepath, path))]) for dirent in dirlist: if dirent.startswith('_cache_tmp') or dirent.startswith('tmp'): gclient_utils.rm_file_or_tree(os.path.join(cachepath, dirent)) elif (dirent.endswith('.lock') and os.path.isfile(os.path.join(cachepath, dirent))): repo_dirs.add(os.path.join(cachepath, dirent[:-5])) unlocked_repos = [] for repo_dir in repo_dirs: if cls.BreakLocks(repo_dir): unlocked_repos.append(repo_dir) return unlocked_repos
def _Scrub(self, target, options): """Scrubs out all changes in the local repo, back to the state of target.""" quiet = [] if not options.verbose: quiet = ['--quiet'] self._Run(['reset', '--hard', target] + quiet, options) if options.force and options.delete_unversioned_trees: # where `target` is a commit that contains both upper and lower case # versions of the same file on a case insensitive filesystem, we are # actually in a broken state here. The index will have both 'a' and 'A', # but only one of them will exist on the disk. To progress, we delete # everything that status thinks is modified. output = self._Capture(['status', '--porcelain'], strip=False) for line in output.splitlines(): # --porcelain (v1) looks like: # XY filename try: filename = line[3:] self.Print('_____ Deleting residual after reset: %r.' % filename) gclient_utils.rm_file_or_tree( os.path.join(self.checkout_path, filename)) except OSError: pass
def bootstrap_repo(self, directory): """Bootstrap the repo from Google Stroage if possible. More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing(). """ python_fallback = False if (sys.platform.startswith('win') and not gclient_utils.FindExecutable('7z')): python_fallback = True elif sys.platform.startswith('darwin'): # The OSX version of unzip doesn't support zip64. python_fallback = True elif not gclient_utils.FindExecutable('unzip'): python_fallback = True gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir) gsutil = Gsutil(self.gsutil_exe, boto_path=None) # Get the most recent version of the zipfile. _, ls_out, _ = gsutil.check_call('ls', gs_folder) ls_out_sorted = sorted(ls_out.splitlines()) if not ls_out_sorted: # This repo is not on Google Storage. return False latest_checkout = ls_out_sorted[-1] # Download zip file to a temporary directory. try: tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath()) self.print('Downloading %s' % latest_checkout) code = gsutil.call('cp', latest_checkout, tempdir) if code: return False filename = os.path.join(tempdir, latest_checkout.split('/')[-1]) # Unpack the file with 7z on Windows, unzip on linux, or fallback. if not python_fallback: if sys.platform.startswith('win'): cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename] else: cmd = ['unzip', filename, '-d', directory] retcode = subprocess.call(cmd) else: try: with zipfile.ZipFile(filename, 'r') as f: f.printdir() f.extractall(directory) except Exception as e: self.print('Encountered error: %s' % str(e), file=sys.stderr) retcode = 1 else: retcode = 0 finally: # Clean up the downloaded zipfile. # # This is somehow racy on Windows. # Catching OSError because WindowsError isn't portable and # pylint complains. exponential_backoff_retry( lambda: gclient_utils.rm_file_or_tree(tempdir), excs=(OSError,), name='rmtree [%s]' % (tempdir,), printerr=self.print) if retcode: self.print( 'Extracting bootstrap zipfile %s failed.\n' 'Resuming normal operations.' % filename) return False return True
def bootstrap_repo(self, directory): """Bootstrap the repo from Google Storage if possible. More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing(). """ if not self.bootstrap_bucket: return False python_fallback = ((sys.platform.startswith('win') and not gclient_utils.FindExecutable('7z')) or (not gclient_utils.FindExecutable('unzip')) or ('ZIP64_SUPPORT' not in subprocess.check_output( ["unzip", "-v"]))) gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir) gsutil = Gsutil(self.gsutil_exe, boto_path=None) # Get the most recent version of the zipfile. _, ls_out, ls_err = gsutil.check_call('ls', gs_folder) def compare_filenames(a, b): # |a| and |b| look like gs://.../.../9999.zip. They both have the same # gs://bootstrap_bucket/basedir/ prefix because they come from the same # `gsutil ls`. # This function only compares the numeral parts before .zip. regex_pattern = r'/(\d+)\.zip$' match_a = re.search(regex_pattern, a) match_b = re.search(regex_pattern, b) if (match_a is not None) and (match_b is not None): num_a = int(match_a.group(1)) num_b = int(match_b.group(1)) return cmp(num_a, num_b) # If it doesn't match the format, fallback to string comparison. return cmp(a, b) ls_out_sorted = sorted(ls_out.splitlines(), cmp=compare_filenames) if not ls_out_sorted: # This repo is not on Google Storage. self.print('No bootstrap file for %s found in %s, stderr:\n %s' % (self.mirror_path, self.bootstrap_bucket, ' '.join( (ls_err or '').splitlines(True)))) return False latest_checkout = ls_out_sorted[-1] # Download zip file to a temporary directory. try: tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath()) self.print('Downloading %s' % latest_checkout) with self.print_duration_of('download'): code = gsutil.call('cp', latest_checkout, tempdir) if code: return False filename = os.path.join(tempdir, latest_checkout.split('/')[-1]) # Unpack the file with 7z on Windows, unzip on linux, or fallback. with self.print_duration_of('unzip'): if not python_fallback: if sys.platform.startswith('win'): cmd = [ '7z', 'x', '-o%s' % directory, '-tzip', filename ] else: cmd = ['unzip', filename, '-d', directory] retcode = subprocess.call(cmd) else: try: with zipfile.ZipFile(filename, 'r') as f: f.printdir() f.extractall(directory) except Exception as e: self.print('Encountered error: %s' % str(e), file=sys.stderr) retcode = 1 else: retcode = 0 finally: # Clean up the downloaded zipfile. # # This is somehow racy on Windows. # Catching OSError because WindowsError isn't portable and # pylint complains. exponential_backoff_retry( lambda: gclient_utils.rm_file_or_tree(tempdir), excs=(OSError, ), name='rmtree [%s]' % (tempdir, ), printerr=self.print) if retcode: self.print('Extracting bootstrap zipfile %s failed.\n' 'Resuming normal operations.' % filename) return False return True
def bootstrap_repo(self, directory): """Bootstrap the repo from Google Storage if possible. More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing(). """ if not self.bootstrap_bucket: return False python_fallback = ( (sys.platform.startswith('win') and not gclient_utils.FindExecutable('7z')) or (not gclient_utils.FindExecutable('unzip')) or ('ZIP64_SUPPORT' not in subprocess.check_output(["unzip", "-v"])) ) gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir) gsutil = Gsutil(self.gsutil_exe, boto_path=None) # Get the most recent version of the zipfile. _, ls_out, ls_err = gsutil.check_call('ls', gs_folder) ls_out_sorted = sorted(ls_out.splitlines()) if not ls_out_sorted: # This repo is not on Google Storage. self.print('No bootstrap file for %s found in %s, stderr:\n %s' % (self.mirror_path, self.bootstrap_bucket, ' '.join((ls_err or '').splitlines(True)))) return False latest_checkout = ls_out_sorted[-1] # Download zip file to a temporary directory. try: tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath()) self.print('Downloading %s' % latest_checkout) with self.print_duration_of('download'): code = gsutil.call('cp', latest_checkout, tempdir) if code: return False filename = os.path.join(tempdir, latest_checkout.split('/')[-1]) # Unpack the file with 7z on Windows, unzip on linux, or fallback. with self.print_duration_of('unzip'): if not python_fallback: if sys.platform.startswith('win'): cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename] else: cmd = ['unzip', filename, '-d', directory] retcode = subprocess.call(cmd) else: try: with zipfile.ZipFile(filename, 'r') as f: f.printdir() f.extractall(directory) except Exception as e: self.print('Encountered error: %s' % str(e), file=sys.stderr) retcode = 1 else: retcode = 0 finally: # Clean up the downloaded zipfile. # # This is somehow racy on Windows. # Catching OSError because WindowsError isn't portable and # pylint complains. exponential_backoff_retry( lambda: gclient_utils.rm_file_or_tree(tempdir), excs=(OSError,), name='rmtree [%s]' % (tempdir,), printerr=self.print) if retcode: self.print( 'Extracting bootstrap zipfile %s failed.\n' 'Resuming normal operations.' % filename) return False return True