def _CheckoutOne(self, opt, project, lock, pm, err_event, err_results):
"""Checkout work tree for one project
Args:
opt: Program options returned from optparse. See _Options().
project: Project object for the project to checkout.
lock: Lock for accessing objects that are shared amongst multiple
_CheckoutWorker() threads.
pm: Instance of a Project object. We will call pm.update() (with our
lock held).
err_event: We'll set this event in the case of an error (after printing
out info about the error).
err_results: A list of strings, paths to git repos where checkout
failed.
Returns:
Whether the fetch was successful.
"""
# We'll set to true once we've locked the lock.
did_lock = False
# Encapsulate everything in a try/except/finally so that:
# - We always set err_event in the case of an exception.
# - We always make sure we unlock the lock if we locked it.
start = time.time()
syncbuf = SyncBuffer(self.manifest.manifestProject.config,
detach_head=opt.detach_head)
success = False
try:
try:
project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
# Lock around all the rest of the code, since printing, updating a set
# and Progress.update() are not thread safe.
lock.acquire()
success = syncbuf.Finish()
did_lock = True
if not success:
err_event.set()
print("error: Cannot checkout %s" % (project.name),
file=sys.stderr)
raise _CheckoutError()
pm.update(msg=project.name)
except _CheckoutError:
pass
except Exception as e:
print(
"error: Cannot checkout %s: %s: %s" %
(project.name, type(e).__name__, str(e)),
file=sys.stderr,
)
err_event.set()
raise
finally:
if did_lock:
if not success:
err_results.append(project.relpath)
lock.release()
finish = time.time()
self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, start,
finish, success)
return success
def _SyncManifest(self, opt):
m = self.manifest.manifestProject
is_new = not m.Exists
if is_new:
if not opt.manifest_url:
print('fatal: manifest url (-u) is required.', file=sys.stderr)
sys.exit(1)
if not opt.quiet:
print('Downloading manifest from %s' %
(GitConfig.ForUser().UrlInsteadOf(opt.manifest_url),),
file=sys.stderr)
# The manifest project object doesn't keep track of the path on the
# server where this git is located, so let's save that here.
mirrored_manifest_git = None
if opt.reference:
manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:]
mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path)
if not mirrored_manifest_git.endswith(".git"):
mirrored_manifest_git += ".git"
if not os.path.exists(mirrored_manifest_git):
mirrored_manifest_git = os.path.join(opt.reference,
'.repo/manifests.git')
m._InitGitDir(mirror_git=mirrored_manifest_git)
self._ConfigureDepth(opt)
# Set the remote URL before the remote branch as we might need it below.
if opt.manifest_url:
r = m.GetRemote(m.remote.name)
r.url = opt.manifest_url
r.ResetFetch()
r.Save()
if opt.manifest_branch:
m.revisionExpr = opt.manifest_branch
else:
if is_new:
default_branch = m.ResolveRemoteHead()
if default_branch is None:
# If the remote doesn't have HEAD configured, default to master.
default_branch = 'refs/heads/master'
m.revisionExpr = default_branch
else:
m.PreSync()
groups = re.split(r'[,\s]+', opt.groups)
all_platforms = ['linux', 'darwin', 'windows']
platformize = lambda x: 'platform-' + x
if opt.platform == 'auto':
if (not opt.mirror and
not m.config.GetString('repo.mirror') == 'true'):
groups.append(platformize(platform.system().lower()))
elif opt.platform == 'all':
groups.extend(map(platformize, all_platforms))
elif opt.platform in all_platforms:
groups.append(platformize(opt.platform))
elif opt.platform != 'none':
print('fatal: invalid platform flag', file=sys.stderr)
sys.exit(1)
groups = [x for x in groups if x]
groupstr = ','.join(groups)
if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system().lower():
groupstr = None
m.config.SetString('manifest.groups', groupstr)
if opt.reference:
m.config.SetString('repo.reference', opt.reference)
if opt.dissociate:
m.config.SetString('repo.dissociate', 'true')
if opt.worktree:
if opt.mirror:
print('fatal: --mirror and --worktree are incompatible',
file=sys.stderr)
sys.exit(1)
if opt.submodules:
print('fatal: --submodules and --worktree are incompatible',
file=sys.stderr)
sys.exit(1)
m.config.SetString('repo.worktree', 'true')
if is_new:
m.use_git_worktrees = True
print('warning: --worktree is experimental!', file=sys.stderr)
if opt.archive:
if is_new:
m.config.SetString('repo.archive', 'true')
else:
print('fatal: --archive is only supported when initializing a new '
'workspace.', file=sys.stderr)
print('Either delete the .repo folder in this workspace, or initialize '
'in another location.', file=sys.stderr)
sys.exit(1)
if opt.mirror:
if is_new:
m.config.SetString('repo.mirror', 'true')
else:
print('fatal: --mirror is only supported when initializing a new '
'workspace.', file=sys.stderr)
print('Either delete the .repo folder in this workspace, or initialize '
'in another location.', file=sys.stderr)
sys.exit(1)
if opt.partial_clone:
if opt.mirror:
print('fatal: --mirror and --partial-clone are mutually exclusive',
file=sys.stderr)
sys.exit(1)
m.config.SetString('repo.partialclone', 'true')
if opt.clone_filter:
m.config.SetString('repo.clonefilter', opt.clone_filter)
else:
opt.clone_filter = None
if opt.clone_bundle is None:
opt.clone_bundle = False if opt.partial_clone else True
else:
m.config.SetString('repo.clonebundle', 'true' if opt.clone_bundle else 'false')
if opt.submodules:
m.config.SetString('repo.submodules', 'true')
if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose,
clone_bundle=opt.clone_bundle,
current_branch_only=opt.current_branch_only,
tags=opt.tags, submodules=opt.submodules,
clone_filter=opt.clone_filter):
r = m.GetRemote(m.remote.name)
print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
# Better delete the manifest git dir if we created it; otherwise next
# time (when user fixes problems) we won't go through the "is_new" logic.
if is_new:
platform_utils.rmtree(m.gitdir)
sys.exit(1)
if opt.manifest_branch:
m.MetaBranchSwitch(submodules=opt.submodules)
syncbuf = SyncBuffer(m.config)
m.Sync_LocalHalf(syncbuf, submodules=opt.submodules)
syncbuf.Finish()
if is_new or m.CurrentBranch is None:
if not m.StartBranch('default'):
print('fatal: cannot create default in manifest', file=sys.stderr)
sys.exit(1)
def _SyncManifest(self, opt):
m = self.manifest.manifestProject
is_new = not m.Exists
if is_new:
if not opt.manifest_url:
print >> sys.stderr, 'fatal: manifest url (-u) is required.'
sys.exit(1)
if not opt.quiet:
print >> sys.stderr, 'Getting manifest ...'
print >> sys.stderr, ' from %s' % opt.manifest_url
m._InitGitDir()
self._ApplyOptions(opt, is_new)
if opt.manifest_url:
r = m.GetRemote(m.remote.name)
r.url = opt.manifest_url
r.ResetFetch()
r.Save()
if opt.reference:
m.config.SetString('repo.reference', opt.reference)
if opt.mirror:
if is_new:
m.config.SetString('repo.mirror', 'true')
m.config.ClearCache()
else:
print >> sys.stderr, 'fatal: --mirror not supported on existing client'
sys.exit(1)
if not m.Sync_NetworkHalf():
r = m.GetRemote(m.remote.name)
print >> sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
sys.exit(1)
if is_new and SubmoduleManifest.IsBare(m):
new = self.GetManifest(reparse=True, type=SubmoduleManifest)
if m.gitdir != new.manifestProject.gitdir:
os.rename(m.gitdir, new.manifestProject.gitdir)
new = self.GetManifest(reparse=True, type=SubmoduleManifest)
m = new.manifestProject
self._ApplyOptions(opt, is_new)
if not is_new:
# Force the manifest to load if it exists, the old graph
# may be needed inside of _ReloadManifest().
#
self.manifest.projects
syncbuf = SyncBuffer(m.config)
m.Sync_LocalHalf(syncbuf)
syncbuf.Finish()
if isinstance(self.manifest, XmlManifest):
self._LinkManifest(opt.manifest_name)
_ReloadManifest(self)
self._ApplyOptions(opt, is_new)
if not self.manifest.InitBranch():
print >> sys.stderr, 'fatal: cannot create branch in manifest'
sys.exit(1)
def _SyncManifest(self, opt):
m = self.manifest.manifestProject
is_new = not m.Exists
if is_new:
if not opt.manifest_url:
print('fatal: manifest url (-u) is required.', file=sys.stderr)
sys.exit(1)
if not opt.quiet:
print('Get %s' % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url),
file=sys.stderr)
# The manifest project object doesn't keep track of the path on the
# server where this git is located, so let's save that here.
mirrored_manifest_git = None
if opt.reference:
manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:]
mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path)
if not mirrored_manifest_git.endswith(".git"):
mirrored_manifest_git += ".git"
if not os.path.exists(mirrored_manifest_git):
mirrored_manifest_git = os.path.join(opt.reference + '/.repo/manifests.git')
m._InitGitDir(mirror_git=mirrored_manifest_git)
if opt.manifest_branch:
m.revisionExpr = opt.manifest_branch
else:
m.revisionExpr = 'refs/heads/master'
else:
if opt.manifest_branch:
m.revisionExpr = opt.manifest_branch
else:
m.PreSync()
if opt.manifest_url:
r = m.GetRemote(m.remote.name)
r.url = opt.manifest_url
r.ResetFetch()
r.Save()
groups = re.split(r'[,\s]+', opt.groups)
all_platforms = ['linux', 'darwin', 'windows']
platformize = lambda x: 'platform-' + x
if opt.platform == 'auto':
if (not opt.mirror and
not m.config.GetString('repo.mirror') == 'true'):
groups.append(platformize(platform.system().lower()))
elif opt.platform == 'all':
groups.extend(map(platformize, all_platforms))
elif opt.platform in all_platforms:
groups.append(platformize(opt.platform))
elif opt.platform != 'none':
print('fatal: invalid platform flag', file=sys.stderr)
sys.exit(1)
groups = [x for x in groups if x]
groupstr = ','.join(groups)
if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system().lower():
groupstr = None
m.config.SetString('manifest.groups', groupstr)
if opt.reference:
m.config.SetString('repo.reference', opt.reference)
if opt.archive:
if is_new:
m.config.SetString('repo.archive', 'true')
else:
print('fatal: --archive is only supported when initializing a new '
'workspace.', file=sys.stderr)
print('Either delete the .repo folder in this workspace, or initialize '
'in another location.', file=sys.stderr)
sys.exit(1)
if opt.mirror:
if is_new:
m.config.SetString('repo.mirror', 'true')
else:
print('fatal: --mirror is only supported when initializing a new '
'workspace.', file=sys.stderr)
print('Either delete the .repo folder in this workspace, or initialize '
'in another location.', file=sys.stderr)
sys.exit(1)
if not m.Sync_NetworkHalf(is_new=is_new):
r = m.GetRemote(m.remote.name)
print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
# Better delete the manifest git dir if we created it; otherwise next
# time (when user fixes problems) we won't go through the "is_new" logic.
if is_new:
shutil.rmtree(m.gitdir)
sys.exit(1)
if opt.manifest_branch:
m.MetaBranchSwitch()
syncbuf = SyncBuffer(m.config)
m.Sync_LocalHalf(syncbuf)
syncbuf.Finish()
if is_new or m.CurrentBranch is None:
if not m.StartBranch('default'):
print('fatal: cannot create default in manifest', file=sys.stderr)
sys.exit(1)
def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
if self.jobs > 1:
soft_limit, _ = _rlimit_nofile()
self.jobs = min(self.jobs, (soft_limit - 5) / 3)
if opt.network_only and opt.detach_head:
print('error: cannot combine -n and -d', file=sys.stderr)
sys.exit(1)
if opt.network_only and opt.local_only:
print('error: cannot combine -n and -l', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_sync:
print('error: cannot combine -m and -s', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_tag:
print('error: cannot combine -m and -t', file=sys.stderr)
sys.exit(1)
if opt.manifest_server_username or opt.manifest_server_password:
if not (opt.smart_sync or opt.smart_tag):
print('error: -u and -p may only be combined with -s or -t',
file=sys.stderr)
sys.exit(1)
if None in [
opt.manifest_server_username, opt.manifest_server_password
]:
print('error: both -u and -p must be given', file=sys.stderr)
sys.exit(1)
if opt.manifest_name:
self.manifest.Override(opt.manifest_name)
manifest_name = opt.manifest_name
if opt.smart_sync or opt.smart_tag:
if not self.manifest.manifest_server:
print(
'error: cannot smart sync: no manifest server defined in '
'manifest',
file=sys.stderr)
sys.exit(1)
manifest_server = self.manifest.manifest_server
if not opt.quiet:
print('Using manifest server %s' % manifest_server)
if not '@' in manifest_server:
username = None
password = None
if opt.manifest_server_username and opt.manifest_server_password:
username = opt.manifest_server_username
password = opt.manifest_server_password
else:
try:
info = netrc.netrc()
except IOError:
print(
'.netrc file does not exist or could not be opened',
file=sys.stderr)
else:
try:
parse_result = urllib.parse.urlparse(
manifest_server)
if parse_result.hostname:
username, _account, password = \
info.authenticators(parse_result.hostname)
except TypeError:
# TypeError is raised when the given hostname is not present
# in the .netrc file.
print('No credentials found for %s in .netrc' %
parse_result.hostname,
file=sys.stderr)
except netrc.NetrcParseError as e:
print('Error parsing .netrc file: %s' % e,
file=sys.stderr)
if (username and password):
manifest_server = manifest_server.replace(
'://', '://%s:%s@' % (username, password), 1)
try:
server = xmlrpc.client.Server(manifest_server)
if opt.smart_sync:
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
branch = b.merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
env = os.environ.copy()
if 'SYNC_TARGET' in env:
target = env['SYNC_TARGET']
[success, manifest_str
] = server.GetApprovedManifest(branch, target)
elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
target = '%s-%s' % (env['TARGET_PRODUCT'],
env['TARGET_BUILD_VARIANT'])
[success, manifest_str
] = server.GetApprovedManifest(branch, target)
else:
[success,
manifest_str] = server.GetApprovedManifest(branch)
else:
assert (opt.smart_tag)
[success, manifest_str] = server.GetManifest(opt.smart_tag)
if success:
manifest_name = "smart_sync_override.xml"
manifest_path = os.path.join(
self.manifest.manifestProject.worktree, manifest_name)
try:
f = open(manifest_path, 'w')
try:
f.write(manifest_str)
finally:
f.close()
except IOError:
print('error: cannot write manifest to %s' %
manifest_path,
file=sys.stderr)
sys.exit(1)
self._ReloadManifest(manifest_name)
else:
print('error: manifest server RPC call failed: %s' %
manifest_str,
file=sys.stderr)
sys.exit(1)
except (socket.error, IOError, xmlrpc.client.Fault) as e:
print('error: cannot connect to manifest server %s:\n%s' %
(self.manifest.manifest_server, e),
file=sys.stderr)
sys.exit(1)
except xmlrpc.client.ProtocolError as e:
print('error: cannot connect to manifest server %s:\n%d %s' %
(self.manifest.manifest_server, e.errcode, e.errmsg),
file=sys.stderr)
sys.exit(1)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest, quiet=opt.quiet)
if not opt.local_only:
mp.Sync_NetworkHalf(quiet=opt.quiet,
current_branch_only=opt.current_branch_only,
no_tags=opt.no_tags)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
mp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1)
self._ReloadManifest(manifest_name)
if opt.jobs is None:
self.jobs = self.manifest.default.sync_j
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
self._fetch_times = _FetchTimes(self.manifest)
if not opt.local_only:
to_fetch = []
now = time.time()
if _ONE_DAY_S <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.extend(all_projects)
to_fetch.sort(key=self._fetch_times.Get, reverse=True)
fetched = self._Fetch(to_fetch, opt)
_PostRepoFetch(rp, opt.no_repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
return
# Iteratively fetch missing and/or nested unregistered submodules
previously_missing_set = set()
while True:
self._ReloadManifest(manifest_name)
all_projects = self.GetProjects(
args, missing_ok=True, submodules_ok=opt.fetch_submodules)
missing = []
for project in all_projects:
if project.gitdir not in fetched:
missing.append(project)
if not missing:
break
# Stop us from non-stopped fetching actually-missing repos: If set of
# missing repos has not been changed from last fetch, we break.
missing_set = set(p.name for p in missing)
if previously_missing_set == missing_set:
break
previously_missing_set = missing_set
fetched.update(self._Fetch(missing, opt))
if self.manifest.IsMirror or self.manifest.IsArchive:
# bail out now, we have no working tree
return
if self.UpdateProjectList():
sys.exit(1)
syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head)
pm = Progress('Syncing work tree', len(all_projects))
for project in all_projects:
pm.update()
if project.worktree:
project.Sync_LocalHalf(syncbuf)
pm.end()
print(file=sys.stderr)
if not syncbuf.Finish():
sys.exit(1)
# If there's a notice that's supposed to print at the end of the sync, print
# it now...
if self.manifest.notice:
print(self.manifest.notice)
def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
if self.jobs > 1:
soft_limit, _ = _rlimit_nofile()
self.jobs = min(self.jobs, (soft_limit - 5) / 3)
if opt.network_only and opt.detach_head:
print('error: cannot combine -n and -d', file=sys.stderr)
sys.exit(1)
if opt.network_only and opt.local_only:
print('error: cannot combine -n and -l', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_sync:
print('error: cannot combine -m and -s', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_tag:
print('error: cannot combine -m and -t', file=sys.stderr)
sys.exit(1)
if opt.manifest_server_username or opt.manifest_server_password:
if not (opt.smart_sync or opt.smart_tag):
print('error: -u and -p may only be combined with -s or -t',
file=sys.stderr)
sys.exit(1)
if None in [opt.manifest_server_username, opt.manifest_server_password]:
print('error: both -u and -p must be given', file=sys.stderr)
sys.exit(1)
if opt.manifest_name:
self.manifest.Override(opt.manifest_name)
manifest_name = opt.manifest_name
smart_sync_manifest_name = "smart_sync_override.xml"
smart_sync_manifest_path = os.path.join(
self.manifest.manifestProject.worktree, smart_sync_manifest_name)
if opt.smart_sync or opt.smart_tag:
if not self.manifest.manifest_server:
print('error: cannot smart sync: no manifest server defined in '
'manifest', file=sys.stderr)
sys.exit(1)
manifest_server = self.manifest.manifest_server
if not opt.quiet:
print('Using manifest server %s' % manifest_server)
if not '@' in manifest_server:
username = None
password = None
if opt.manifest_server_username and opt.manifest_server_password:
username = opt.manifest_server_username
password = opt.manifest_server_password
else:
try:
info = netrc.netrc()
except IOError:
# .netrc file does not exist or could not be opened
pass
else:
try:
parse_result = urllib.parse.urlparse(manifest_server)
if parse_result.hostname:
auth = info.authenticators(parse_result.hostname)
if auth:
username, _account, password = auth
else:
print('No credentials found for %s in .netrc'
% parse_result.hostname, file=sys.stderr)
except netrc.NetrcParseError as e:
print('Error parsing .netrc file: %s' % e, file=sys.stderr)
if (username and password):
manifest_server = manifest_server.replace('://', '://%s:%s@' %
(username, password),
1)
transport = PersistentTransport(manifest_server)
if manifest_server.startswith('persistent-'):
manifest_server = manifest_server[len('persistent-'):]
try:
server = xmlrpc.client.Server(manifest_server, transport=transport)
if opt.smart_sync:
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
branch = b.merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
env = os.environ.copy()
if 'SYNC_TARGET' in env:
target = env['SYNC_TARGET']
[success, manifest_str] = server.GetApprovedManifest(branch, target)
elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
target = '%s-%s' % (env['TARGET_PRODUCT'],
env['TARGET_BUILD_VARIANT'])
[success, manifest_str] = server.GetApprovedManifest(branch, target)
else:
[success, manifest_str] = server.GetApprovedManifest(branch)
else:
assert(opt.smart_tag)
[success, manifest_str] = server.GetManifest(opt.smart_tag)
if success:
manifest_name = smart_sync_manifest_name
try:
f = open(smart_sync_manifest_path, 'w')
try:
f.write(manifest_str)
finally:
f.close()
except IOError as e:
print('error: cannot write manifest to %s:\n%s'
% (smart_sync_manifest_path, e),
file=sys.stderr)
sys.exit(1)
self._ReloadManifest(manifest_name)
else:
print('error: manifest server RPC call failed: %s' %
manifest_str, file=sys.stderr)
sys.exit(1)
except (socket.error, IOError, xmlrpc.client.Fault) as e:
print('error: cannot connect to manifest server %s:\n%s'
% (self.manifest.manifest_server, e), file=sys.stderr)
sys.exit(1)
except xmlrpc.client.ProtocolError as e:
print('error: cannot connect to manifest server %s:\n%d %s'
% (self.manifest.manifest_server, e.errcode, e.errmsg),
file=sys.stderr)
sys.exit(1)
else: # Not smart sync or smart tag mode
if os.path.isfile(smart_sync_manifest_path):
try:
platform_utils.remove(smart_sync_manifest_path)
except OSError as e:
print('error: failed to remove existing smart sync override manifest: %s' %
e, file=sys.stderr)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest, quiet=opt.quiet)
if not opt.local_only:
start = time.time()
success = mp.Sync_NetworkHalf(quiet=opt.quiet,
current_branch_only=opt.current_branch_only,
no_tags=opt.no_tags,
optimized_fetch=opt.optimized_fetch,
submodules=self.manifest.HasSubmodules)
finish = time.time()
self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
start, finish, success)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
start = time.time()
mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
clean = syncbuf.Finish()
self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
start, time.time(), clean)
if not clean:
sys.exit(1)
self._ReloadManifest(manifest_name)
if opt.jobs is None:
self.jobs = self.manifest.default.sync_j
if self.gitc_manifest:
gitc_manifest_projects = self.GetProjects(args,
missing_ok=True)
gitc_projects = []
opened_projects = []
for project in gitc_manifest_projects:
if project.relpath in self.gitc_manifest.paths and \
self.gitc_manifest.paths[project.relpath].old_revision:
opened_projects.append(project.relpath)
else:
gitc_projects.append(project.relpath)
if not args:
gitc_projects = None
if gitc_projects != [] and not opt.local_only:
print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name)
if manifest_name:
manifest.Override(manifest_name)
else:
manifest.Override(self.manifest.manifestFile)
gitc_utils.generate_gitc_manifest(self.gitc_manifest,
manifest,
gitc_projects)
print('GITC client successfully synced.')
# The opened projects need to be synced as normal, therefore we
# generate a new args list to represent the opened projects.
# TODO: make this more reliable -- if there's a project name/path overlap,
# this may choose the wrong project.
args = [os.path.relpath(self.manifest.paths[path].worktree, os.getcwd())
for path in opened_projects]
if not args:
return
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
self._fetch_times = _FetchTimes(self.manifest)
if not opt.local_only:
to_fetch = []
now = time.time()
if _ONE_DAY_S <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.extend(all_projects)
to_fetch.sort(key=self._fetch_times.Get, reverse=True)
fetched = self._Fetch(to_fetch, opt)
_PostRepoFetch(rp, opt.no_repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
return
# Iteratively fetch missing and/or nested unregistered submodules
previously_missing_set = set()
while True:
self._ReloadManifest(manifest_name)
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
missing = []
for project in all_projects:
if project.gitdir not in fetched:
missing.append(project)
if not missing:
break
# Stop us from non-stopped fetching actually-missing repos: If set of
# missing repos has not been changed from last fetch, we break.
missing_set = set(p.name for p in missing)
if previously_missing_set == missing_set:
break
previously_missing_set = missing_set
fetched.update(self._Fetch(missing, opt))
if self.manifest.IsMirror or self.manifest.IsArchive:
# bail out now, we have no working tree
return
if self.UpdateProjectList():
sys.exit(1)
syncbuf = SyncBuffer(mp.config,
detach_head = opt.detach_head)
pm = Progress('Syncing work tree', len(all_projects))
for project in all_projects:
pm.update()
if project.worktree:
start = time.time()
project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
start, time.time(), syncbuf.Recently())
pm.end()
print(file=sys.stderr)
if not syncbuf.Finish():
sys.exit(1)
# If there's a notice that's supposed to print at the end of the sync, print
# it now...
if self.manifest.notice:
print(self.manifest.notice)
def Execute(self, opt, args):
if opt.network_only and opt.detach_head:
print >> sys.stderr, 'error: cannot combine -n and -d'
sys.exit(1)
if opt.network_only and opt.local_only:
print >> sys.stderr, 'error: cannot combine -n and -l'
sys.exit(1)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest)
all = self.GetProjects(args, missing_ok=True)
if not opt.local_only:
to_fetch = []
now = time.time()
if (24 * 60 * 60) <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.append(mp)
to_fetch.extend(all)
fetched = self._Fetch(to_fetch)
_PostRepoFetch(rp, opt.no_repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
return
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
mp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1)
self.manifest._Unload()
all = self.GetProjects(args, missing_ok=True)
missing = []
for project in all:
if project.gitdir not in fetched:
missing.append(project)
self._Fetch(missing)
if self.manifest.IsMirror:
# bail out now, we have no working tree
return
if self.UpdateProjectList():
sys.exit(1)
syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head)
pm = Progress('Syncing work tree', len(all))
for project in all:
pm.update()
if project.worktree:
project.Sync_LocalHalf(syncbuf)
pm.end()
print >> sys.stderr
if not syncbuf.Finish():
sys.exit(1)
def Execute(self, opt, args):
if opt.jobs:
self.jobs = 3
if opt.network_only and opt.detach_head:
print >> sys.stderr, 'error: cannot combine -n and -d'
sys.exit(1)
if opt.network_only and opt.local_only:
print >> sys.stderr, 'error: cannot combine -n and -l'
sys.exit(1)
if opt.smart_sync:
if not self.manifest.manifest_server:
print >>sys.stderr, \
'error: cannot smart sync: no manifest server defined in manifest'
sys.exit(1)
try:
server = xmlrpclib.Server(self.manifest.manifest_server)
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
branch = b.merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
env = os.environ.copy()
if (env.has_key('TARGET_PRODUCT')
and env.has_key('TARGET_BUILD_VARIANT')):
target = '%s-%s' % (env['TARGET_PRODUCT'],
env['TARGET_BUILD_VARIANT'])
[success, manifest_str
] = server.GetApprovedManifest(branch, target)
else:
[success,
manifest_str] = server.GetApprovedManifest(branch)
if success:
manifest_name = "smart_sync_override.xml"
manifest_path = os.path.join(
self.manifest.manifestProject.worktree, manifest_name)
try:
f = open(manifest_path, 'w')
try:
f.write(manifest_str)
finally:
f.close()
except IOError:
print >>sys.stderr, 'error: cannot write manifest to %s' % \
manifest_path
sys.exit(1)
self.manifest.Override(manifest_name)
else:
print >> sys.stderr, 'error: %s' % manifest_str
sys.exit(1)
except socket.error:
print >> sys.stderr, 'error: cannot connect to manifest server %s' % (
self.manifest.manifest_server)
sys.exit(1)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest)
if not opt.local_only:
mp.Sync_NetworkHalf(quiet=opt.quiet)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
mp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1)
self.manifest._Unload()
all = self.GetProjects(args, missing_ok=True)
if not opt.local_only:
to_fetch = []
now = time.time()
if (24 * 60 * 60) <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.extend(all)
fetched = self._Fetch(to_fetch, opt)
_PostRepoFetch(rp, True)
if opt.network_only:
# bail out now; the rest touches the working tree
return
self.manifest._Unload()
all = self.GetProjects(args, missing_ok=True)
missing = []
for project in all:
if project.gitdir not in fetched:
missing.append(project)
self._Fetch(missing, opt)
if self.manifest.IsMirror:
# bail out now, we have no working tree
return
if self.UpdateProjectList():
sys.exit(1)
syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head)
pm = Progress('Syncing work tree', len(all))
for project in all:
pm.update()
if project.worktree:
project.Sync_LocalHalf(syncbuf)
pm.end()
print >> sys.stderr
if not syncbuf.Finish():
sys.exit(1)
# If there's a notice that's supposed to print at the end of the sync, print
# it now...
if self.manifest.notice:
print self.manifest.notice
def _SyncManifest(self, opt):
m = self.manifest.manifestProject
is_new = not m.Exists
# If repo has already been initialized, we take -u with the absence of
# --standalone-manifest to mean "transition to a standard repo set up",
# which necessitates starting fresh.
# If --standalone-manifest is set, we always tear everything down and start
# anew.
if not is_new:
was_standalone_manifest = m.config.GetString('manifest.standalone')
if was_standalone_manifest and not opt.manifest_url:
print('fatal: repo was initialized with a standlone manifest, '
'cannot be re-initialized without --manifest-url/-u')
sys.exit(1)
if opt.standalone_manifest or (was_standalone_manifest
and opt.manifest_url):
m.config.ClearCache()
if m.gitdir and os.path.exists(m.gitdir):
platform_utils.rmtree(m.gitdir)
if m.worktree and os.path.exists(m.worktree):
platform_utils.rmtree(m.worktree)
is_new = not m.Exists
if is_new:
if not opt.manifest_url:
print('fatal: manifest url is required.', file=sys.stderr)
sys.exit(1)
if not opt.quiet:
print('Downloading manifest from %s' %
(GitConfig.ForUser().UrlInsteadOf(opt.manifest_url), ),
file=sys.stderr)
# The manifest project object doesn't keep track of the path on the
# server where this git is located, so let's save that here.
mirrored_manifest_git = None
if opt.reference:
manifest_git_path = urllib.parse.urlparse(
opt.manifest_url).path[1:]
mirrored_manifest_git = os.path.join(opt.reference,
manifest_git_path)
if not mirrored_manifest_git.endswith(".git"):
mirrored_manifest_git += ".git"
if not os.path.exists(mirrored_manifest_git):
mirrored_manifest_git = os.path.join(
opt.reference, '.repo/manifests.git')
m._InitGitDir(mirror_git=mirrored_manifest_git)
# If standalone_manifest is set, mark the project as "standalone" -- we'll
# still do much of the manifests.git set up, but will avoid actual syncs to
# a remote.
standalone_manifest = False
if opt.standalone_manifest:
standalone_manifest = True
m.config.SetString('manifest.standalone', opt.manifest_url)
elif not opt.manifest_url and not opt.manifest_branch:
# If -u is set and --standalone-manifest is not, then we're not in
# standalone mode. Otherwise, use config to infer what we were in the last
# init.
standalone_manifest = bool(
m.config.GetString('manifest.standalone'))
if not standalone_manifest:
m.config.SetString('manifest.standalone', None)
self._ConfigureDepth(opt)
# Set the remote URL before the remote branch as we might need it below.
if opt.manifest_url:
r = m.GetRemote(m.remote.name)
r.url = opt.manifest_url
r.ResetFetch()
r.Save()
if not standalone_manifest:
if opt.manifest_branch:
if opt.manifest_branch == 'HEAD':
opt.manifest_branch = m.ResolveRemoteHead()
if opt.manifest_branch is None:
print('fatal: unable to resolve HEAD', file=sys.stderr)
sys.exit(1)
m.revisionExpr = opt.manifest_branch
else:
if is_new:
default_branch = m.ResolveRemoteHead()
if default_branch is None:
# If the remote doesn't have HEAD configured, default to master.
default_branch = 'refs/heads/master'
m.revisionExpr = default_branch
else:
m.PreSync()
groups = re.split(r'[,\s]+', opt.groups)
all_platforms = ['linux', 'darwin', 'windows']
platformize = lambda x: 'platform-' + x
if opt.platform == 'auto':
if (not opt.mirror
and not m.config.GetString('repo.mirror') == 'true'):
groups.append(platformize(platform.system().lower()))
elif opt.platform == 'all':
groups.extend(map(platformize, all_platforms))
elif opt.platform in all_platforms:
groups.append(platformize(opt.platform))
elif opt.platform != 'none':
print('fatal: invalid platform flag', file=sys.stderr)
sys.exit(1)
groups = [x for x in groups if x]
groupstr = ','.join(groups)
if opt.platform == 'auto' and groupstr == self.manifest.GetDefaultGroupsStr(
):
groupstr = None
m.config.SetString('manifest.groups', groupstr)
if opt.reference:
m.config.SetString('repo.reference', opt.reference)
if opt.dissociate:
m.config.SetBoolean('repo.dissociate', opt.dissociate)
if opt.worktree:
if opt.mirror:
print('fatal: --mirror and --worktree are incompatible',
file=sys.stderr)
sys.exit(1)
if opt.submodules:
print('fatal: --submodules and --worktree are incompatible',
file=sys.stderr)
sys.exit(1)
m.config.SetBoolean('repo.worktree', opt.worktree)
if is_new:
m.use_git_worktrees = True
print('warning: --worktree is experimental!', file=sys.stderr)
if opt.archive:
if is_new:
m.config.SetBoolean('repo.archive', opt.archive)
else:
print(
'fatal: --archive is only supported when initializing a new '
'workspace.',
file=sys.stderr)
print(
'Either delete the .repo folder in this workspace, or initialize '
'in another location.',
file=sys.stderr)
sys.exit(1)
if opt.mirror:
if is_new:
m.config.SetBoolean('repo.mirror', opt.mirror)
else:
print(
'fatal: --mirror is only supported when initializing a new '
'workspace.',
file=sys.stderr)
print(
'Either delete the .repo folder in this workspace, or initialize '
'in another location.',
file=sys.stderr)
sys.exit(1)
if opt.partial_clone is not None:
if opt.mirror:
print(
'fatal: --mirror and --partial-clone are mutually exclusive',
file=sys.stderr)
sys.exit(1)
m.config.SetBoolean('repo.partialclone', opt.partial_clone)
if opt.clone_filter:
m.config.SetString('repo.clonefilter', opt.clone_filter)
elif m.config.GetBoolean('repo.partialclone'):
opt.clone_filter = m.config.GetString('repo.clonefilter')
else:
opt.clone_filter = None
if opt.partial_clone_exclude is not None:
m.config.SetString('repo.partialcloneexclude',
opt.partial_clone_exclude)
if opt.clone_bundle is None:
opt.clone_bundle = False if opt.partial_clone else True
else:
m.config.SetBoolean('repo.clonebundle', opt.clone_bundle)
if opt.submodules:
m.config.SetBoolean('repo.submodules', opt.submodules)
if opt.use_superproject is not None:
m.config.SetBoolean('repo.superproject', opt.use_superproject)
if standalone_manifest:
if is_new:
manifest_name = 'default.xml'
manifest_data = fetch.fetch_file(opt.manifest_url,
verbose=opt.verbose)
dest = os.path.join(m.worktree, manifest_name)
os.makedirs(os.path.dirname(dest), exist_ok=True)
with open(dest, 'wb') as f:
f.write(manifest_data)
return
if not m.Sync_NetworkHalf(
is_new=is_new,
quiet=opt.quiet,
verbose=opt.verbose,
clone_bundle=opt.clone_bundle,
current_branch_only=opt.current_branch_only,
tags=opt.tags,
submodules=opt.submodules,
clone_filter=opt.clone_filter,
partial_clone_exclude=self.manifest.PartialCloneExclude):
r = m.GetRemote(m.remote.name)
print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
# Better delete the manifest git dir if we created it; otherwise next
# time (when user fixes problems) we won't go through the "is_new" logic.
if is_new:
platform_utils.rmtree(m.gitdir)
sys.exit(1)
if opt.manifest_branch:
m.MetaBranchSwitch(submodules=opt.submodules)
syncbuf = SyncBuffer(m.config)
m.Sync_LocalHalf(syncbuf, submodules=opt.submodules)
syncbuf.Finish()
if is_new or m.CurrentBranch is None:
if not m.StartBranch('default'):
print('fatal: cannot create default in manifest',
file=sys.stderr)
sys.exit(1)
def _SyncManifest(self, opt):
m = self.manifest.manifestProject
is_new = not m.Exists
if is_new:
if not opt.manifest_url:
print >> sys.stderr, 'fatal: manifest url (-u) is required.'
sys.exit(1)
if not opt.quiet:
print >>sys.stderr, 'Get %s' \
% GitConfig.ForUser().UrlInsteadOf(opt.manifest_url)
m._InitGitDir()
if opt.manifest_branch:
m.revisionExpr = opt.manifest_branch
else:
m.revisionExpr = 'refs/heads/master'
else:
if opt.manifest_branch:
m.revisionExpr = opt.manifest_branch
else:
m.PreSync()
if opt.manifest_url:
r = m.GetRemote(m.remote.name)
r.url = opt.manifest_url
r.ResetFetch()
r.Save()
groups = re.split('[,\s]+', opt.groups)
all_platforms = ['linux', 'darwin']
platformize = lambda x: 'platform-' + x
if opt.platform == 'auto':
if (not opt.mirror
and not m.config.GetString('repo.mirror') == 'true'):
groups.append(platformize(platform.system().lower()))
elif opt.platform == 'all':
groups.extend(map(platformize, all_platforms))
elif opt.platform in all_platforms:
groups.extend(platformize(opt.platform))
elif opt.platform != 'none':
print >> sys.stderr, 'fatal: invalid platform flag'
sys.exit(1)
groups = [x for x in groups if x]
groupstr = ','.join(groups)
if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system(
).lower():
groupstr = None
m.config.SetString('manifest.groups', groupstr)
if opt.reference:
m.config.SetString('repo.reference', opt.reference)
if opt.mirror:
if is_new:
m.config.SetString('repo.mirror', 'true')
else:
print >> sys.stderr, 'fatal: --mirror not supported on existing client'
sys.exit(1)
if not m.Sync_NetworkHalf(is_new=is_new):
r = m.GetRemote(m.remote.name)
print >> sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
# Better delete the manifest git dir if we created it; otherwise next
# time (when user fixes problems) we won't go through the "is_new" logic.
if is_new:
shutil.rmtree(m.gitdir)
sys.exit(1)
if opt.manifest_branch:
m.MetaBranchSwitch(opt.manifest_branch)
syncbuf = SyncBuffer(m.config)
m.Sync_LocalHalf(syncbuf)
syncbuf.Finish()
if is_new or m.CurrentBranch is None:
if not m.StartBranch('default'):
print >> sys.stderr, 'fatal: cannot create default in manifest'
sys.exit(1)
