def Execute(self, opt, args):
git_require(MIN_GIT_VERSION, fail=True)
self._SyncManifest(opt)
self._LinkManifest(opt.manifest_name)
if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
if opt.config_name or self._ShouldConfigureUser():
self._ConfigureUser()
self._ConfigureColor()
self._DisplayResult()
def Execute(self, opt, args):
git_require(MIN_GIT_VERSION_HARD, fail=True)
if not git_require(MIN_GIT_VERSION_SOFT):
print(
'repo: warning: git-%s+ will soon be required; please upgrade your '
'version of git to maintain support.' %
('.'.join(str(x) for x in MIN_GIT_VERSION_SOFT), ),
file=sys.stderr)
opt.quiet = opt.output_mode is False
opt.verbose = opt.output_mode is True
rp = self.manifest.repoProject
# Handle new --repo-url requests.
if opt.repo_url:
remote = rp.GetRemote('origin')
remote.url = opt.repo_url
remote.Save()
# Handle new --repo-rev requests.
if opt.repo_rev:
wrapper = Wrapper()
remote_ref, rev = wrapper.check_repo_rev(
rp.gitdir,
opt.repo_rev,
repo_verify=opt.repo_verify,
quiet=opt.quiet)
branch = rp.GetBranch('default')
branch.merge = remote_ref
rp.work_git.reset('--hard', rev)
branch.Save()
if opt.worktree:
# Older versions of git supported worktree, but had dangerous gc bugs.
git_require((2, 15, 0),
fail=True,
msg='git gc worktree corruption')
self._SyncManifest(opt)
self._LinkManifest(opt.manifest_name)
if self.manifest.manifestProject.config.GetBoolean(
'repo.superproject'):
self._CloneSuperproject(opt)
if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
if opt.config_name or self._ShouldConfigureUser(opt):
self._ConfigureUser(opt)
self._ConfigureColor()
self._DisplayResult(opt)
def Execute(self, opt, args):
git_require(MIN_GIT_VERSION, fail=True)
self._SyncManifest(opt)
if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
self._ConfigureUser(opt)
self._ConfigureColor(opt)
if self.manifest.IsMirror:
type = 'mirror '
else:
type = ''
print ''
print 'repo %sinitialized in %s' % (type, self.manifest.topdir)
def Execute(self, opt, args):
git_require(MIN_GIT_VERSION, fail=True)
self._SyncManifest(opt)
if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
self._ConfigureUser()
self._ConfigureColor()
if self.manifest.IsMirror:
type = 'mirror '
else:
type = ''
print ''
print 'repo %sinitialized in %s' % (type, self.manifest.topdir)
def _GCProjects(self, projects, opt, err_event):
gc_gitdirs = {}
for project in projects:
# Make sure pruning never kicks in with shared projects.
if (not project.use_git_worktrees and len(
project.manifest.GetProjectsWithName(project.name)) > 1):
print("%s: Shared project %s found, disabling pruning." %
(project.relpath, project.name))
if git_require((2, 7, 0)):
project.EnableRepositoryExtension("preciousObjects")
else:
# This isn't perfect, but it's the best we can do with old git.
print(
"%s: WARNING: shared projects are unreliable when using old "
"versions of git; please upgrade to git-2.7.0+." %
(project.relpath, ),
file=sys.stderr,
)
project.config.SetString("gc.pruneExpire", "never")
gc_gitdirs[project.gitdir] = project.bare_git
if multiprocessing:
cpu_count = multiprocessing.cpu_count()
else:
cpu_count = 1
jobs = min(self.jobs, cpu_count)
if jobs < 2:
for bare_git in gc_gitdirs.values():
bare_git.gc("--auto")
return
config = {"pack.threads": cpu_count // jobs if cpu_count > jobs else 1}
threads = set()
sem = _threading.Semaphore(jobs)
def GC(bare_git):
try:
try:
bare_git.gc("--auto", config=config)
except GitError:
err_event.set()
except Exception:
err_event.set()
raise
finally:
sem.release()
for bare_git in gc_gitdirs.values():
if err_event.isSet() and opt.fail_fast:
break
sem.acquire()
t = _threading.Thread(target=GC, args=(bare_git, ))
t.daemon = True
threads.add(t)
t.start()
for t in threads:
t.join()
def _Fetch(self):
"""Fetches a local copy of a superproject for the manifest based on |_remote_url|.
Returns:
True if fetch is successful, or False.
"""
if not os.path.exists(self._work_git):
self._LogWarning(f'git fetch missing directory: {self._work_git}')
return False
if not git_require((2, 28, 0)):
self._LogWarning(
'superproject requires a git version 2.28 or later')
return False
cmd = [
'fetch', self._remote_url, '--depth', '1', '--force', '--no-tags',
'--filter', 'blob:none'
]
if self._branch:
cmd += [self._branch + ':' + self._branch]
p = GitCommand(None,
cmd,
cwd=self._work_git,
capture_stdout=True,
capture_stderr=True)
retval = p.Wait()
if retval:
self._LogWarning(f'git fetch call failed, command: git {cmd}, '
f'return code: {retval}, stderr: {p.stderr}')
return False
return True
def setUpClass(cls):
# Create a repo to operate on, but do it once per-class.
cls.GIT_DIR = tempfile.mkdtemp(prefix='repo-rev-tests')
run_git = wrapper.Wrapper().run_git
remote = os.path.join(cls.GIT_DIR, 'remote')
os.mkdir(remote)
# Tests need to assume, that main is default branch at init,
# which is not supported in config until 2.28.
if git_command.git_require((2, 28, 0)):
initstr = '--initial-branch=main'
else:
# Use template dir for init.
templatedir = tempfile.mkdtemp(prefix='.test-template')
with open(os.path.join(templatedir, 'HEAD'), 'w') as fp:
fp.write('ref: refs/heads/main\n')
initstr = '--template=' + templatedir
run_git('init', initstr, cwd=remote)
run_git('commit', '--allow-empty', '-minit', cwd=remote)
run_git('branch', 'stable', cwd=remote)
run_git('tag', 'v1.0', cwd=remote)
run_git('commit', '--allow-empty', '-m2nd commit', cwd=remote)
cls.REV_LIST = run_git('rev-list', 'HEAD',
cwd=remote).stdout.splitlines()
run_git('init', cwd=cls.GIT_DIR)
run_git('fetch',
remote,
'+refs/heads/*:refs/remotes/origin/*',
cwd=cls.GIT_DIR)
def Execute(self, opt, args):
git_require(MIN_GIT_VERSION_HARD, fail=True)
if not git_require(MIN_GIT_VERSION_SOFT):
print('repo: warning: git-%s+ will soon be required; please upgrade your '
'version of git to maintain support.'
% ('.'.join(str(x) for x in MIN_GIT_VERSION_SOFT),),
file=sys.stderr)
self._SyncManifest(opt)
self._LinkManifest(opt.manifest_name)
if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
if opt.config_name or self._ShouldConfigureUser():
self._ConfigureUser()
self._ConfigureColor()
self._DisplayResult()
def Execute(self, opt, args):
git_require(MIN_GIT_VERSION, fail=True)
if opt.reference:
opt.reference = os.path.expanduser(opt.reference)
self._SyncManifest(opt)
self._LinkManifest(opt.manifest_name)
if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
if opt.config_name or self._ShouldConfigureUser():
self._ConfigureUser()
self._ConfigureColor()
self._ConfigureDepth(opt)
self._DisplayResult()
def _GCProjects(self, projects):
gc_gitdirs = {}
for project in projects:
if len(project.manifest.GetProjectsWithName(project.name)) > 1:
print('Shared project %s found, disabling pruning.' %
project.name)
project.bare_git.config('--replace-all', 'gc.pruneExpire',
'never')
gc_gitdirs[project.gitdir] = project.bare_git
has_dash_c = git_require((1, 7, 2))
if multiprocessing and has_dash_c:
cpu_count = multiprocessing.cpu_count()
else:
cpu_count = 1
jobs = min(self.jobs, cpu_count)
if jobs < 2:
for bare_git in gc_gitdirs.values():
bare_git.gc('--auto')
return
config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
threads = set()
sem = _threading.Semaphore(jobs)
err_event = _threading.Event()
def GC(bare_git):
try:
try:
bare_git.gc('--auto', config=config)
except GitError:
err_event.set()
except:
err_event.set()
raise
finally:
sem.release()
for bare_git in gc_gitdirs.values():
if err_event.isSet():
break
sem.acquire()
t = _threading.Thread(target=GC, args=(bare_git, ))
t.daemon = True
threads.add(t)
t.start()
for t in threads:
t.join()
if err_event.isSet():
print('\nerror: Exited sync due to gc errors', file=sys.stderr)
sys.exit(1)
def _GCProjects(self, projects):
gc_gitdirs = {}
for project in projects:
if len(project.manifest.GetProjectsWithName(project.name)) > 1:
print('Shared project %s found, disabling pruning.' % project.name)
project.bare_git.config('--replace-all', 'gc.pruneExpire', 'never')
gc_gitdirs[project.gitdir] = project.bare_git
has_dash_c = git_require((1, 7, 2))
if multiprocessing and has_dash_c:
cpu_count = multiprocessing.cpu_count()
else:
cpu_count = 1
jobs = min(self.jobs, cpu_count)
if jobs < 2:
for bare_git in gc_gitdirs.values():
bare_git.gc('--auto')
return
config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
threads = set()
sem = _threading.Semaphore(jobs)
err_event = _threading.Event()
def GC(bare_git):
try:
try:
bare_git.gc('--auto', config=config)
except GitError:
err_event.set()
except:
err_event.set()
raise
finally:
sem.release()
for bare_git in gc_gitdirs.values():
if err_event.isSet():
break
sem.acquire()
t = _threading.Thread(target=GC, args=(bare_git,))
t.daemon = True
threads.add(t)
t.start()
for t in threads:
t.join()
if err_event.isSet():
print('\nerror: Exited sync due to gc errors', file=sys.stderr)
sys.exit(1)
def Execute(self, opt, args):
git_require(MIN_GIT_VERSION, fail=True)
if opt.reference:
opt.reference = os.path.expanduser(opt.reference)
# Check this here, else manifest will be tagged "not new" and init won't be
# possible anymore without removing the .repo/manifests directory.
if opt.archive and opt.mirror:
print('fatal: --mirror and --archive cannot be used together.',
file=sys.stderr)
sys.exit(1)
self._SyncManifest(opt)
self._LinkManifest(opt.manifest_name)
if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
if opt.config_name or self._ShouldConfigureUser():
self._ConfigureUser()
self._ConfigureColor()
self._DisplayResult()
def _GCProjects(self, projects):
gitdirs = {}
for project in projects:
gitdirs[project.gitdir] = project.bare_git
has_dash_c = git_require((1, 7, 2))
if multiprocessing and has_dash_c:
cpu_count = multiprocessing.cpu_count()
else:
cpu_count = 1
jobs = min(self.jobs, cpu_count)
if jobs < 2:
for bare_git in gitdirs.values():
bare_git.gc("--auto")
return
config = {"pack.threads": cpu_count / jobs if cpu_count > jobs else 1}
threads = set()
sem = _threading.Semaphore(jobs)
err_event = _threading.Event()
def GC(bare_git):
try:
try:
bare_git.gc("--auto", config=config)
except GitError:
err_event.set()
except:
err_event.set()
raise
finally:
sem.release()
for bare_git in gitdirs.values():
if err_event.isSet():
break
sem.acquire()
t = _threading.Thread(target=GC, args=(bare_git,))
t.daemon = True
threads.add(t)
t.start()
for t in threads:
t.join()
if err_event.isSet():
print("\nerror: Exited sync due to gc errors", file=sys.stderr)
sys.exit(1)
def Execute(self, opt, args):
git_require(MIN_GIT_VERSION, fail=True)
if opt.reference:
opt.reference = os.path.expanduser(opt.reference)
self._SyncManifest(opt)
self._LinkManifest(opt.manifest_name)
if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
if opt.config_name or self._ShouldConfigureUser():
self._ConfigureUser()
self._ConfigureColor()
self._ConfigureDepth(opt)
if self.manifest.IsMirror:
init_type = 'mirror '
else:
init_type = ''
print ''
print 'repo %sinitialized in %s' % (init_type, self.manifest.topdir)
def Execute(self, opt, args):
git_require(MIN_GIT_VERSION, fail=True)
if opt.reference:
opt.reference = os.path.expanduser(opt.reference)
self._SyncManifest(opt)
self._LinkManifest(opt.manifest_name)
if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
if opt.config_name or self._ShouldConfigureUser():
self._ConfigureUser()
self._ConfigureColor()
self._ConfigureDepth(opt)
if self.manifest.IsMirror:
init_type = 'mirror '
else:
init_type = ''
print ''
print 'repo %sinitialized in %s' % (init_type, self.manifest.topdir)
def Execute(self, opt, args):
git_require(MIN_GIT_VERSION_HARD, fail=True)
if not git_require(MIN_GIT_VERSION_SOFT):
print('repo: warning: git-%s+ will soon be required; please upgrade your '
'version of git to maintain support.'
% ('.'.join(str(x) for x in MIN_GIT_VERSION_SOFT),),
file=sys.stderr)
opt.quiet = opt.output_mode is False
opt.verbose = opt.output_mode is True
if opt.worktree:
# Older versions of git supported worktree, but had dangerous gc bugs.
git_require((2, 15, 0), fail=True, msg='git gc worktree corruption')
self._SyncManifest(opt)
self._LinkManifest(opt.manifest_name)
if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
if opt.config_name or self._ShouldConfigureUser(opt):
self._ConfigureUser(opt)
self._ConfigureColor()
self._DisplayResult(opt)
def Execute(self, opt, args):
git_require(MIN_GIT_VERSION, fail=True)
if opt.reference:
opt.reference = os.path.expanduser(opt.reference)
# Check this here, else manifest will be tagged "not new" and init won't be
# possible anymore without removing the .repo/manifests directory.
if opt.archive and opt.mirror:
print('fatal: --mirror and --archive cannot be used together.',
file=sys.stderr)
sys.exit(1)
self._SyncManifest(opt)
self._LinkManifest(opt.manifest_name)
if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
if opt.config_name or self._ShouldConfigureUser():
self._ConfigureUser()
self._ConfigureColor()
self._ConfigureDepth(opt)
self._DisplayResult()
def _GCProjects(self, projects):
has_dash_c = git_require((1, 7, 2))
if multiprocessing and has_dash_c:
cpu_count = multiprocessing.cpu_count()
else:
cpu_count = 1
jobs = min(self.jobs, cpu_count)
if jobs < 2:
for project in projects:
project.bare_git.gc('--auto')
return
config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
threads = set()
sem = _threading.Semaphore(jobs)
err_event = _threading.Event()
def GC(project):
try:
try:
project.bare_git.gc('--auto', config=config)
except GitError:
err_event.set()
except:
err_event.set()
raise
finally:
sem.release()
for project in projects:
if err_event.isSet():
break
sem.acquire()
t = _threading.Thread(target=GC, args=(project, ))
t.daemon = True
threads.add(t)
t.start()
for t in threads:
t.join()
if err_event.isSet():
print('\nerror: Exited sync due to gc errors', file=sys.stderr)
sys.exit(1)
def TempGitTree():
"""Create a new empty git checkout for testing."""
# TODO(vapier): Convert this to tempfile.TemporaryDirectory once we drop
# Python 2 support entirely.
try:
tempdir = tempfile.mkdtemp(prefix='repo-tests')
# Tests need to assume, that main is default branch at init,
# which is not supported in config until 2.28.
cmd = ['git', 'init']
if git_command.git_require((2, 28, 0)):
cmd += ['--initial-branch=main']
else:
# Use template dir for init.
templatedir = tempfile.mkdtemp(prefix='.test-template')
with open(os.path.join(templatedir, 'HEAD'), 'w') as fp:
fp.write('ref: refs/heads/main\n')
cmd += ['--template=', templatedir]
subprocess.check_call(cmd, cwd=tempdir)
yield tempdir
finally:
platform_utils.rmtree(tempdir)
def test_older_nonfatal(self):
"""Test non-fatal require calls with old versions."""
self.assertFalse(git_command.git_require((2, )))
self.assertFalse(git_command.git_require((1, 3)))
self.assertFalse(git_command.git_require((1, 2, 4)))
self.assertFalse(git_command.git_require((1, 2, 3, 5)))
def test_older_fatal_msg(self):
"""Test fatal require calls with old versions and message."""
with self.assertRaises(SystemExit) as e:
git_command.git_require((2, ), fail=True, msg='so sad')
self.assertNotEqual(0, e.code)
def test_newer_nonfatal(self):
"""Test non-fatal require calls with newer versions."""
self.assertTrue(git_command.git_require((0, )))
self.assertTrue(git_command.git_require((1, 0)))
self.assertTrue(git_command.git_require((1, 2, 0)))
self.assertTrue(git_command.git_require((1, 2, 3, 0)))
def test_equal_nonfatal(self):
"""Test require calls with equal values."""
self.assertTrue(git_command.git_require((1, 2, 3, 4), fail=False))
self.assertTrue(git_command.git_require((1, 2, 3, 4), fail=True))
def _GCProjects(self, projects, opt, err_event):
pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)
pm.update(inc=0, msg='prescan')
tidy_dirs = {}
for project in projects:
# Make sure pruning never kicks in with shared projects.
if (not project.use_git_worktrees and
len(project.manifest.GetProjectsWithName(project.name)) > 1):
if not opt.quiet:
print('\r%s: Shared project %s found, disabling pruning.' %
(project.relpath, project.name))
if git_require((2, 7, 0)):
project.EnableRepositoryExtension('preciousObjects')
else:
# This isn't perfect, but it's the best we can do with old git.
print('\r%s: WARNING: shared projects are unreliable when using old '
'versions of git; please upgrade to git-2.7.0+.'
% (project.relpath,),
file=sys.stderr)
project.config.SetString('gc.pruneExpire', 'never')
project.config.SetString('gc.autoDetach', 'false')
# Only call git gc once per objdir, but call pack-refs for the remainder.
if project.objdir not in tidy_dirs:
tidy_dirs[project.objdir] = (
True, # Run a full gc.
project.bare_git,
)
elif project.gitdir not in tidy_dirs:
tidy_dirs[project.gitdir] = (
False, # Do not run a full gc; just run pack-refs.
project.bare_git,
)
cpu_count = os.cpu_count()
jobs = min(self.jobs, cpu_count)
if jobs < 2:
for (run_gc, bare_git) in tidy_dirs.values():
pm.update(msg=bare_git._project.name)
if run_gc:
bare_git.gc('--auto')
else:
bare_git.pack_refs()
pm.end()
return
config = {'pack.threads': cpu_count // jobs if cpu_count > jobs else 1}
threads = set()
sem = _threading.Semaphore(jobs)
def tidy_up(run_gc, bare_git):
pm.start(bare_git._project.name)
try:
try:
if run_gc:
bare_git.gc('--auto', config=config)
else:
bare_git.pack_refs(config=config)
except GitError:
err_event.set()
except Exception:
err_event.set()
raise
finally:
pm.finish(bare_git._project.name)
sem.release()
for (run_gc, bare_git) in tidy_dirs.values():
if err_event.is_set() and opt.fail_fast:
break
sem.acquire()
t = _threading.Thread(target=tidy_up, args=(run_gc, bare_git,))
t.daemon = True
threads.add(t)
t.start()
for t in threads:
t.join()
pm.end()
def Execute(self, opt, args):
out = GrepColoring(self.manifest.manifestProject.config)
cmd_argv = ['grep']
if out.is_on and git_require((1, 6, 3)):
cmd_argv.append('--color')
cmd_argv.extend(getattr(opt, 'cmd_argv', []))
if '-e' not in cmd_argv:
if not args:
self.Usage()
cmd_argv.append('-e')
cmd_argv.append(args[0])
args = args[1:]
projects = self.GetProjects(args)
full_name = False
if len(projects) > 1:
cmd_argv.append('--full-name')
full_name = True
have_rev = False
if opt.revision:
if '--cached' in cmd_argv:
print('fatal: cannot combine --cached and --revision', file=sys.stderr)
sys.exit(1)
have_rev = True
cmd_argv.extend(opt.revision)
cmd_argv.append('--')
bad_rev = False
have_match = False
for project in projects:
p = GitCommand(project,
cmd_argv,
bare=False,
capture_stdout=True,
capture_stderr=True)
if p.Wait() != 0:
# no results
#
if p.stderr:
if have_rev and 'fatal: ambiguous argument' in p.stderr:
bad_rev = True
else:
out.project('--- project %s ---' % project.relpath)
out.nl()
out.write("%s", p.stderr)
out.nl()
continue
have_match = True
# We cut the last element, to avoid a blank line.
#
r = portable.stream2str(p.stdout).split('\n')
r = r[0:-1]
if have_rev and full_name:
for line in r:
rev, line = line.split(':', 1)
out.write("%s", rev)
out.write(':')
out.project(project.relpath)
out.write('/')
out.write("%s", line)
out.nl()
elif full_name:
for line in r:
out.project(project.relpath)
out.write('/')
out.write("%s", line)
out.nl()
else:
for line in r:
print(line)
if have_match:
sys.exit(0)
elif have_rev and bad_rev:
for r in opt.revision:
print("error: can't search revision %s" % r, file=sys.stderr)
sys.exit(1)
else:
sys.exit(1)
def Execute(self, opt, args):
out = GrepColoring(self.manifest.manifestProject.config)
cmd_argv = ['grep']
if out.is_on and git_require((1, 6, 3)):
cmd_argv.append('--color')
cmd_argv.extend(getattr(opt, 'cmd_argv', []))
if '-e' not in cmd_argv:
if not args:
self.Usage()
cmd_argv.append('-e')
cmd_argv.append(args[0])
args = args[1:]
projects = self.GetProjects(args)
full_name = False
if len(projects) > 1:
cmd_argv.append('--full-name')
full_name = True
have_rev = False
if opt.revision:
if '--cached' in cmd_argv:
print('fatal: cannot combine --cached and --revision',
file=sys.stderr)
sys.exit(1)
have_rev = True
cmd_argv.extend(opt.revision)
cmd_argv.append('--')
bad_rev = False
have_match = False
for project in projects:
p = GitCommand(project,
cmd_argv,
bare=False,
capture_stdout=True,
capture_stderr=True)
if p.Wait() != 0:
# no results
#
if p.stderr:
if have_rev and 'fatal: ambiguous argument' in p.stderr:
bad_rev = True
else:
out.project('--- project %s ---' % project.relpath)
out.nl()
out.write("%s", p.stderr)
out.nl()
continue
have_match = True
# We cut the last element, to avoid a blank line.
#
r = portable.stream2str(p.stdout).split('\n')
r = r[0:-1]
if have_rev and full_name:
for line in r:
rev, line = line.split(':', 1)
out.write("%s", rev)
out.write(':')
out.project(project.relpath)
out.write('/')
out.write("%s", line)
out.nl()
elif full_name:
for line in r:
out.project(project.relpath)
out.write('/')
out.write("%s", line)
out.nl()
else:
for line in r:
print(line)
if have_match:
sys.exit(0)
elif have_rev and bad_rev:
for r in opt.revision:
print("error: can't search revision %s" % r, file=sys.stderr)
sys.exit(1)
else:
sys.exit(1)
