def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
if not git.check_ref_format('heads/%s' % nb):
print >>sys.stderr, "error: '%s' is not a valid name" % nb
sys.exit(1)
err = []
projects = []
if not opt.all:
projects = args[1:]
if len(projects) < 1:
print >>sys.stderr, "error: at least one project must be specified"
sys.exit(1)
all = self.GetProjects(projects)
pm = Progress('Starting %s' % nb, len(all))
for project in all:
pm.update()
if not project.StartBranch(nb):
err.append(project)
pm.end()
if err:
for p in err:
print >>sys.stderr,\
"error: %s/: cannot start %s" \
% (p.relpath, nb)
sys.exit(1)
def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
if not git.check_ref_format('heads/%s' % nb):
print >> sys.stderr, "error: '%s' is not a valid name" % nb
sys.exit(1)
nb = args[0]
err = []
all = self.GetProjects(args[1:])
pm = Progress('Abandon %s' % nb, len(all))
for project in all:
pm.update()
if not project.AbandonBranch(nb):
err.append(project)
pm.end()
if err:
if len(err) == len(all):
print >> sys.stderr, 'error: no project has branch %s' % nb
else:
for p in err:
print >>sys.stderr,\
"error: %s/: cannot abandon %s" \
% (p.relpath, nb)
sys.exit(1)
def _Fetch(self, projects):
fetched = set()
pm = Progress('Fetching projects', len(projects))
if self.jobs == 1:
for project in projects:
pm.update()
if project.Sync_NetworkHalf():
fetched.add(project.gitdir)
else:
print >>sys.stderr, 'error: Cannot fetch %s' % project.name
sys.exit(1)
else:
threads = set()
lock = _threading.Lock()
sem = _threading.Semaphore(self.jobs)
for project in projects:
sem.acquire()
t = _threading.Thread(target = self._FetchHelper,
args = (project, lock, fetched, pm, sem))
threads.add(t)
t.start()
for t in threads:
t.join()
pm.end()
return fetched
def Execute(self, opt, args):
nb = args[0]
err = []
success = []
all_projects = self.GetProjects(args[1:])
pm = Progress("Checkout %s" % nb, len(all_projects))
for project in all_projects:
pm.update()
status = project.CheckoutBranch(nb)
if status is not None:
if status:
success.append(project)
else:
err.append(project)
pm.end()
if err:
for p in err:
print("error: %s/: cannot checkout %s" % (p.relpath, nb),
file=sys.stderr)
sys.exit(1)
elif not success:
print("error: no project has branch %s" % nb, file=sys.stderr)
sys.exit(1)
def Execute(self, opt, args):
if not opt.all and not args:
self.Usage()
if not opt.all:
nb = args[0]
if not git.check_ref_format('heads/%s' % nb):
print("error: '%s' is not a valid name" % nb, file=sys.stderr)
sys.exit(1)
else:
args.insert(0,None)
nb = "'All local branches'"
err = defaultdict(list)
success = defaultdict(list)
all_projects = self.GetProjects(args[1:])
pm = Progress('Abandon %s' % nb, len(all_projects))
for project in all_projects:
pm.update()
if opt.all:
branches = project.GetBranches().keys()
else:
branches = [nb]
for name in branches:
status = project.AbandonBranch(name)
if status is not None:
if status:
success[name].append(project)
else:
err[name].append(project)
pm.end()
width = 25
for name in branches:
if width < len(name):
width = len(name)
if err:
for br in err.keys():
err_msg = "error: cannot abandon %s" %br
print(err_msg, file=sys.stderr)
for proj in err[br]:
print(' '*len(err_msg) + " | %s" % proj.relpath, file=sys.stderr)
sys.exit(1)
elif not success:
print('error: no project has local branch(es) : %s' % nb,
file=sys.stderr)
sys.exit(1)
else:
print('Abandoned branches:', file=sys.stderr)
for br in success.keys():
if len(all_projects) > 1 and len(all_projects) == len(success[br]):
result = "all project"
else:
result = "%s" % (
('\n'+' '*width + '| ').join(p.relpath for p in success[br]))
print("%s%s| %s\n" % (br,' '*(width-len(br)), result),file=sys.stderr)
def _Checkout(self, all_projects, opt, err_event, err_results):
"""Checkout projects listed in all_projects
Args:
all_projects: List of all projects that should be checked out.
opt: Program options returned from optparse. See _Options().
err_event: We'll set this event in the case of an error (after printing
out info about the error).
err_results: A list of strings, paths to git repos where checkout
failed.
"""
# Perform checkouts in multiple threads when we are using partial clone.
# Without partial clone, all needed git objects are already downloaded,
# in this situation it's better to use only one process because the checkout
# would be mostly disk I/O; with partial clone, the objects are only
# downloaded when demanded (at checkout time), which is similar to the
# Sync_NetworkHalf case and parallelism would be helpful.
if self.manifest.CloneFilter:
syncjobs = self.jobs
else:
syncjobs = 1
lock = _threading.Lock()
pm = Progress("Checking out projects", len(all_projects))
threads = set()
sem = _threading.Semaphore(syncjobs)
for project in all_projects:
# Check for any errors before running any more tasks.
# ...we'll let existing threads finish, though.
if err_event.isSet() and opt.fail_fast:
break
sem.acquire()
if project.worktree:
kwargs = dict(
opt=opt,
sem=sem,
project=project,
lock=lock,
pm=pm,
err_event=err_event,
err_results=err_results,
)
if syncjobs > 1:
t = _threading.Thread(target=self._CheckoutWorker,
kwargs=kwargs)
# Ensure that Ctrl-C will not freeze the repo process.
t.daemon = True
threads.add(t)
t.start()
else:
self._CheckoutWorker(**kwargs)
for t in threads:
t.join()
pm.end()
def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
if not git.check_ref_format('heads/%s' % nb):
print >>sys.stderr, "error: '%s' is not a valid name" % nb
sys.exit(1)
nb = args[0]
err = []
all = self.GetProjects(args[1:])
pm = Progress('Abandon %s' % nb, len(all))
for project in all:
pm.update()
if not project.AbandonBranch(nb):
err.append(project)
pm.end()
if err:
if len(err) == len(all):
print >>sys.stderr, 'error: no project has branch %s' % nb
else:
for p in err:
print >>sys.stderr,\
"error: %s/: cannot abandon %s" \
% (p.relpath, nb)
sys.exit(1)
def _Fetch(self, projects, opt):
fetched = set()
pm = Progress("Fetching projects", len(projects))
if self.jobs == 1:
for project in projects:
pm.update()
if project.Sync_NetworkHalf(quiet=opt.quiet):
fetched.add(project.gitdir)
else:
print >> sys.stderr, "error: Cannot fetch %s" % project.name
if opt.force_broken:
print >> sys.stderr, "warn: --force-broken, continuing to sync"
else:
sys.exit(1)
else:
threads = set()
lock = _threading.Lock()
sem = _threading.Semaphore(self.jobs)
for project in projects:
sem.acquire()
t = _threading.Thread(target=self._FetchHelper, args=(opt, project, lock, fetched, pm, sem))
threads.add(t)
t.start()
for t in threads:
t.join()
pm.end()
for project in projects:
project.bare_git.gc("--auto")
return fetched
def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
if not git.check_ref_format('heads/%s' % nb):
print("error: '%s' is not a valid name" % nb, file=sys.stderr)
sys.exit(1)
err = []
projects = []
if not opt.all:
projects = args[1:]
if len(projects) < 1:
print("error: at least one project must be specified", file=sys.stderr)
sys.exit(1)
all_projects = self.GetProjects(projects)
pm = Progress('Starting %s' % nb, len(all_projects))
for project in all_projects:
pm.update()
# If the current revision is a specific SHA1 then we can't push back
# to it so substitute the manifest default revision instead.
if IsId(project.revisionExpr):
project.revisionExpr = self.manifest.default.revisionExpr
if not project.StartBranch(nb):
err.append(project)
pm.end()
if err:
for p in err:
print("error: %s/: cannot start %s" % (p.relpath, nb),
file=sys.stderr)
sys.exit(1)
def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
err = []
success = []
all = self.GetProjects(args[1:])
pm = Progress('Checkout %s' % nb, len(all))
for project in all:
pm.update()
status = project.CheckoutBranch(nb)
if status is not None:
if status:
success.append(project)
else:
err.append(project)
pm.end()
if err:
for p in err:
print >>sys.stderr,\
"error: %s/: cannot checkout %s" \
% (p.relpath, nb)
sys.exit(1)
elif not success:
print >>sys.stderr, 'error: no project has branch %s' % nb
sys.exit(1)
def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
if not git.check_ref_format('heads/%s' % nb):
print >> sys.stderr, "error: '%s' is not a valid name" % nb
sys.exit(1)
err = []
projects = []
if not opt.all:
projects = args[1:]
if len(projects) < 1:
print >> sys.stderr, "error: at least one project must be specified"
sys.exit(1)
all = self.GetProjects(projects)
pm = Progress('Starting %s' % nb, len(all))
for project in all:
pm.update()
if not project.StartBranch(nb):
err.append(project)
pm.end()
if err:
for p in err:
print >>sys.stderr,\
"error: %s/: cannot start %s" \
% (p.relpath, nb)
sys.exit(1)
def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
if not git.check_ref_format('heads/%s' % nb):
print("error: '%s' is not a valid name" % nb, file=sys.stderr)
sys.exit(1)
err = []
projects = []
if not opt.all:
projects = args[1:]
if len(projects) < 1:
projects = ['.',] # start it in the local project by default
all_projects = self.GetProjects(projects)
pm = Progress('Starting %s' % nb, len(all_projects))
for project in all_projects:
pm.update()
# If the current revision is a specific SHA1 then we can't push back
# to it so substitute the manifest default revision instead.
if IsId(project.revisionExpr):
project.revisionExpr = self.manifest.default.revisionExpr
if not project.StartBranch(nb):
err.append(project)
pm.end()
if err:
for p in err:
print("error: %s/: cannot start %s" % (p.relpath, nb),
file=sys.stderr)
sys.exit(1)
def _Fetch(self, projects, opt):
fetched = set()
pm = Progress('Fetching projects', len(projects))
if self.jobs == 1:
for project in projects:
pm.update()
if project.Sync_NetworkHalf(quiet=opt.quiet):
fetched.add(project.gitdir)
else:
print >> sys.stderr, 'error: Cannot fetch %s' % project.name
if opt.force_broken:
print >> sys.stderr, 'warn: --force-broken, continuing to sync'
else:
sys.exit(1)
else:
threads = set()
lock = _threading.Lock()
sem = _threading.Semaphore(self.jobs)
for project in projects:
sem.acquire()
t = _threading.Thread(target=self._FetchHelper,
args=(opt, project, lock, fetched, pm,
sem))
threads.add(t)
t.start()
for t in threads:
t.join()
pm.end()
for project in projects:
project.bare_git.gc('--auto')
return fetched
def Execute(self, opt, args):
nb = args[0]
err = []
success = []
all_projects = self.GetProjects(args[1:])
def _ProcessResults(results):
for status, project in results:
if status is not None:
if status:
success.append(project)
else:
err.append(project)
pm.update()
pm = Progress('Checkout %s' % nb, len(all_projects))
# NB: Multiprocessing is heavy, so don't spin it up for one job.
if len(all_projects) == 1 or opt.jobs == 1:
_ProcessResults(self._ExecuteOne(nb, x) for x in all_projects)
else:
with multiprocessing.Pool(opt.jobs) as pool:
results = pool.imap_unordered(
functools.partial(self._ExecuteOne, nb), all_projects,
chunksize=WORKER_BATCH_SIZE)
_ProcessResults(results)
pm.end()
if err:
for p in err:
print("error: %s/: cannot checkout %s" % (p.relpath, nb),
file=sys.stderr)
sys.exit(1)
elif not success:
print('error: no project has branch %s' % nb, file=sys.stderr)
sys.exit(1)
def _Fetch(self, projects, opt):
fetched = set()
pm = Progress('Fetching projects', len(projects))
if self.jobs == 1:
for project in projects:
pm.update()
if not opt.quiet:
print('Fetching project %s' % project.name)
if project.Sync_NetworkHalf(
quiet=opt.quiet,
current_branch_only=opt.current_branch_only,
clone_bundle=not opt.no_clone_bundle,
no_tags=opt.no_tags):
fetched.add(project.gitdir)
else:
print('error: Cannot fetch %s' % project.name, file=sys.stderr)
if opt.force_broken:
print('warn: --force-broken, continuing to sync', file=sys.stderr)
else:
sys.exit(1)
else:
threads = set()
lock = _threading.Lock()
sem = _threading.Semaphore(self.jobs)
err_event = _threading.Event()
for project in projects:
# Check for any errors before starting any new threads.
# ...we'll let existing threads finish, though.
if err_event.isSet():
break
sem.acquire()
t = _threading.Thread(target = self._FetchHelper,
args = (opt,
project,
lock,
fetched,
pm,
sem,
err_event))
# Ensure that Ctrl-C will not freeze the repo process.
t.daemon = True
threads.add(t)
t.start()
for t in threads:
t.join()
# If we saw an error, exit with code 1 so that other scripts can check.
if err_event.isSet():
print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
sys.exit(1)
pm.end()
self._fetch_times.Save()
self._GCProjects(projects)
return fetched
def _Checkout(self, all_projects, opt):
"""Checkout projects listed in all_projects
Args:
all_projects: List of all projects that should be checked out.
opt: Program options returned from optparse. See _Options().
"""
# Perform checkouts in multiple threads when we are using partial clone.
# Without partial clone, all needed git objects are already downloaded,
# in this situation it's better to use only one process because the checkout
# would be mostly disk I/O; with partial clone, the objects are only
# downloaded when demanded (at checkout time), which is similar to the
# Sync_NetworkHalf case and parallelism would be helpful.
if self.manifest.CloneFilter:
syncjobs = self.jobs
else:
syncjobs = 1
lock = _threading.Lock()
pm = Progress('Syncing work tree', len(all_projects))
threads = set()
sem = _threading.Semaphore(syncjobs)
err_event = _threading.Event()
for project in all_projects:
# Check for any errors before running any more tasks.
# ...we'll let existing threads finish, though.
if err_event.isSet() and not opt.force_broken:
break
sem.acquire()
if project.worktree:
kwargs = dict(opt=opt,
sem=sem,
project=project,
lock=lock,
pm=pm,
err_event=err_event)
if syncjobs > 1:
t = _threading.Thread(target=self._CheckoutWorker,
kwargs=kwargs)
# Ensure that Ctrl-C will not freeze the repo process.
t.daemon = True
threads.add(t)
t.start()
else:
self._CheckoutWorker(**kwargs)
for t in threads:
t.join()
pm.end()
# If we saw an error, exit with code 1 so that other scripts can check.
if err_event.isSet():
print('\nerror: Exited sync due to checkout errors',
file=sys.stderr)
sys.exit(1)
def _Fetch(self, projects, opt):
fetched = set()
pm = Progress('Fetching projects', len(projects))
if self.jobs == 1:
for project in projects:
pm.update()
if not opt.quiet:
print('Fetching project %s' % project.name)
if project.Sync_NetworkHalf(
quiet=opt.quiet,
current_branch_only=opt.current_branch_only,
clone_bundle=not opt.no_clone_bundle,
no_tags=opt.no_tags):
fetched.add(project.gitdir)
else:
print('error: Cannot fetch %s' % project.name, file=sys.stderr)
if opt.force_broken:
print('warn: --force-broken, continuing to sync', file=sys.stderr)
else:
sys.exit(1)
else:
threads = set()
lock = _threading.Lock()
sem = _threading.Semaphore(self.jobs)
err_event = _threading.Event()
for project in projects:
# Check for any errors before starting any new threads.
# ...we'll let existing threads finish, though.
if err_event.isSet():
break
sem.acquire()
t = _threading.Thread(target = self._FetchHelper,
args = (opt,
project,
lock,
fetched,
pm,
sem,
err_event))
# Ensure that Ctrl-C will not freeze the repo process.
t.daemon = True
threads.add(t)
t.start()
for t in threads:
t.join()
# If we saw an error, exit with code 1 so that other scripts can check.
if err_event.isSet():
print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
sys.exit(1)
pm.end()
self._fetch_times.Save()
self._GCProjects(projects)
return fetched
def Execute(self, opt, args):
nb = args[0]
err = defaultdict(list)
success = defaultdict(list)
all_projects = self.GetProjects(args[1:])
def _ProcessResults(states):
for (results, project) in states:
for branch, status in results.items():
if status:
success[branch].append(project)
else:
err[branch].append(project)
pm.update()
pm = Progress('Abandon %s' % nb, len(all_projects))
# NB: Multiprocessing is heavy, so don't spin it up for one job.
if len(all_projects) == 1 or opt.jobs == 1:
_ProcessResults(self._ExecuteOne(opt, nb, x) for x in all_projects)
else:
with multiprocessing.Pool(opt.jobs) as pool:
states = pool.imap_unordered(functools.partial(
self._ExecuteOne, opt, nb),
all_projects,
chunksize=WORKER_BATCH_SIZE)
_ProcessResults(states)
pm.end()
width = max(
itertools.chain([25],
(len(x) for x in itertools.chain(success, err))))
if err:
for br in err.keys():
err_msg = "error: cannot abandon %s" % br
print(err_msg, file=sys.stderr)
for proj in err[br]:
print(' ' * len(err_msg) + " | %s" % proj.relpath,
file=sys.stderr)
sys.exit(1)
elif not success:
print('error: no project has local branch(es) : %s' % nb,
file=sys.stderr)
sys.exit(1)
else:
# Everything below here is displaying status.
if opt.quiet:
return
print('Abandoned branches:')
for br in success.keys():
if len(all_projects) > 1 and len(all_projects) == len(
success[br]):
result = "all project"
else:
result = "%s" % (
('\n' + ' ' * width + '| ').join(p.relpath
for p in success[br]))
print("%s%s| %s\n" % (br, ' ' * (width - len(br)), result))
def Execute(self, opt, args):
nb = args[0]
err = defaultdict(list)
success = defaultdict(list)
all_projects = self.GetProjects(args[1:])
pm = Progress('Abandon %s' % nb, len(all_projects))
for project in all_projects:
pm.update()
if opt.all:
branches = list(project.GetBranches().keys())
else:
branches = [nb]
for name in branches:
status = project.AbandonBranch(name)
if status is not None:
if status:
success[name].append(project)
else:
err[name].append(project)
pm.end()
width = 25
for name in branches:
if width < len(name):
width = len(name)
if err:
for br in err.keys():
err_msg = "error: cannot abandon %s" % br
print(err_msg, file=sys.stderr)
for proj in err[br]:
print(' ' * len(err_msg) + " | %s" % proj.relpath,
file=sys.stderr)
sys.exit(1)
elif not success:
print('error: no project has local branch(es) : %s' % nb,
file=sys.stderr)
sys.exit(1)
else:
# Everything below here is displaying status.
if opt.quiet:
return
print('Abandoned branches:')
for br in success.keys():
if len(all_projects) > 1 and len(all_projects) == len(
success[br]):
result = "all project"
else:
result = "%s" % (
('\n' + ' ' * width + '| ').join(p.relpath
for p in success[br]))
print("%s%s| %s\n" % (br, ' ' * (width - len(br)), result))
def _Fetch(self, projects, opt):
fetched = set()
lock = _threading.Lock()
pm = Progress('Fetching projects',
len(projects),
print_newline=not (opt.quiet),
always_print_percentage=opt.quiet)
objdir_project_map = dict()
for project in projects:
objdir_project_map.setdefault(project.objdir, []).append(project)
threads = set()
sem = _threading.Semaphore(self.jobs)
err_event = _threading.Event()
for project_list in objdir_project_map.values():
# Check for any errors before running any more tasks.
# ...we'll let existing threads finish, though.
if err_event.isSet() and opt.fail_fast:
break
sem.acquire()
kwargs = dict(opt=opt,
projects=project_list,
sem=sem,
lock=lock,
fetched=fetched,
pm=pm,
err_event=err_event,
clone_filter=self.manifest.CloneFilter)
if self.jobs > 1:
t = _threading.Thread(target=self._FetchProjectList,
kwargs=kwargs)
# Ensure that Ctrl-C will not freeze the repo process.
t.daemon = True
threads.add(t)
t.start()
else:
self._FetchProjectList(**kwargs)
for t in threads:
t.join()
# If we saw an error, exit with code 1 so that other scripts can check.
if err_event.isSet() and opt.fail_fast:
print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
sys.exit(1)
pm.end()
self._fetch_times.Save()
if not self.manifest.IsArchive:
self._GCProjects(projects)
return fetched
def FromXml_Definition(self, old):
"""Convert another manifest representation to this one.
"""
mp = self.manifestProject
gm = self._modules
gr = self._review
fd = open(os.path.join(mp.worktree, '.gitignore'), 'ab')
fd.write('/.repo\n')
fd.close()
sort_projects = list(old.projects.keys())
sort_projects.sort()
b = mp.GetBranch(mp.CurrentBranch).merge
if b.startswith(R_HEADS):
b = b[len(R_HEADS):]
if old.notice:
gm.SetString('repo.notice', old.notice)
info = []
pm = Progress('Converting manifest', len(sort_projects))
for p in sort_projects:
pm.update()
p = old.projects[p]
gm.SetString('submodule.%s.path' % p.name, p.relpath)
gm.SetString('submodule.%s.url' % p.name, p.remote.url)
if gr.GetString('review.url') is None:
gr.SetString('review.url', p.remote.review)
elif gr.GetString('review.url') != p.remote.review:
gr.SetString('review.%s.url' % p.name, p.remote.review)
r = p.revisionExpr
if r and not IsId(r):
if r.startswith(R_HEADS):
r = r[len(R_HEADS):]
if r == b:
r = '.'
gm.SetString('submodule.%s.revision' % p.name, r)
for c in p.copyfiles:
info.append('Moved %s out of %s' % (c.src, p.relpath))
c._Copy()
p.work_git.rm(c.src)
mp.work_git.add(c.dest)
self.SetRevisionId(p.relpath, p.GetRevisionId())
mp.work_git.add('.gitignore', '.gitmodules', '.review')
pm.end()
for i in info:
print >> sys.stderr, i
def FromXml_Definition(self, old):
"""Convert another manifest representation to this one.
"""
mp = self.manifestProject
gm = self._modules
gr = self._review
fd = open(os.path.join(mp.worktree, '.gitignore'), 'ab')
fd.write('/.repo\n')
fd.close()
sort_projects = list(old.projects.keys())
sort_projects.sort()
b = mp.GetBranch(mp.CurrentBranch).merge
if b.startswith(R_HEADS):
b = b[len(R_HEADS):]
if old.notice:
gm.SetString('repo.notice', old.notice)
info = []
pm = Progress('Converting manifest', len(sort_projects))
for p in sort_projects:
pm.update()
p = old.projects[p]
gm.SetString('submodule.%s.path' % p.name, p.relpath)
gm.SetString('submodule.%s.url' % p.name, p.remote.url)
if gr.GetString('review.url') is None:
gr.SetString('review.url', p.remote.review)
elif gr.GetString('review.url') != p.remote.review:
gr.SetString('review.%s.url' % p.name, p.remote.review)
r = p.revisionExpr
if r and not IsId(r):
if r.startswith(R_HEADS):
r = r[len(R_HEADS):]
if r == b:
r = '.'
gm.SetString('submodule.%s.revision' % p.name, r)
for c in p.copyfiles:
info.append('Moved %s out of %s' % (c.src, p.relpath))
c._Copy()
p.work_git.rm(c.src)
mp.work_git.add(c.dest)
self.SetRevisionId(p.relpath, p.GetRevisionId())
mp.work_git.add('.gitignore', '.gitmodules', '.review')
pm.end()
for i in info:
print >>sys.stderr, i
def _Fetch(self, projects, opt):
fetched = set()
lock = _threading.Lock()
pm = Progress('Fetching projects', len(projects),
print_newline=not(opt.quiet),
always_print_percentage=opt.quiet)
objdir_project_map = dict()
for project in projects:
objdir_project_map.setdefault(project.objdir, []).append(project)
threads = set()
sem = _threading.Semaphore(self.jobs)
err_event = _threading.Event()
for project_list in objdir_project_map.values():
# Check for any errors before running any more tasks.
# ...we'll let existing threads finish, though.
if err_event.isSet() and not opt.force_broken:
break
sem.acquire()
kwargs = dict(opt=opt,
projects=project_list,
sem=sem,
lock=lock,
fetched=fetched,
pm=pm,
err_event=err_event)
if self.jobs > 1:
t = _threading.Thread(target = self._FetchProjectList,
kwargs = kwargs)
# Ensure that Ctrl-C will not freeze the repo process.
t.daemon = True
threads.add(t)
t.start()
else:
self._FetchProjectList(**kwargs)
for t in threads:
t.join()
# If we saw an error, exit with code 1 so that other scripts can check.
if err_event.isSet() and not opt.force_broken:
print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
sys.exit(1)
pm.end()
self._fetch_times.Save()
if not self.manifest.IsArchive:
self._GCProjects(projects)
return fetched
def Execute(self, opt, args):
err = []
projects = []
if not opt.all:
projects = args
if len(projects) < 1:
print("error: at least one project must be specified", file=sys.stderr)
sys.exit(1)
if self.gitc_manifest:
all_projects = self.GetProjects(projects, manifest=self.gitc_manifest,
missing_ok=True)
for project in all_projects:
if project.old_revision:
project.already_synced = True
else:
project.already_synced = False
project.old_revision = project.revisionExpr
project.revisionExpr = None
# Save the GITC manifest.
gitc_utils.save_manifest(self.gitc_manifest)
all_projects = self.GetProjects(projects,
missing_ok=bool(self.gitc_manifest))
pm = Progress('Pushing', len(all_projects))
for project in all_projects:
pm.update()
if self.gitc_manifest:
gitc_project = self.gitc_manifest.paths[project.relpath]
# Pull projects that have not been opened.
if not gitc_project.already_synced:
proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir,
project.relpath)
project.worktree = proj_localdir
if not os.path.exists(proj_localdir):
os.makedirs(proj_localdir)
project.Sync_NetworkHalf()
sync_buf = SyncBuffer(self.manifest.manifestProject.config)
project.Sync_LocalHalf(sync_buf)
project.revisionId = gitc_project.old_revision
if not project.Push():
err.append(project)
pm.end()
if err:
for p in err:
print("error: %s/: cannot push" % p.relpath,
file=sys.stderr)
sys.exit(1)
def _Fetch(self, projects):
fetched = set()
pm = Progress('Fetching projects', len(projects))
for project in projects:
pm.update()
if project.Sync_NetworkHalf():
fetched.add(project.gitdir)
else:
print >> sys.stderr, 'error: Cannot fetch %s' % project.name
sys.exit(1)
pm.end()
return fetched
def _Fetch(self, projects, opt):
fetched = set()
pm = Progress('Fetching projects', len(projects))
if self.jobs == 1:
for project in projects:
pm.update()
if project.Sync_NetworkHalf(quiet=opt.quiet,
current_branch_only=opt.current_branch_only):
fetched.add(project.gitdir)
else:
print >>sys.stderr, 'error: Cannot fetch %s' % project.name
if opt.force_broken:
print >>sys.stderr, 'warn: --force-broken, continuing to sync'
else:
sys.exit(1)
else:
threads = set()
lock = _threading.Lock()
sem = _threading.Semaphore(self.jobs)
err_event = _threading.Event()
for project in projects:
# Check for any errors before starting any new threads.
# ...we'll let existing threads finish, though.
if err_event.isSet():
break
sem.acquire()
t = _threading.Thread(target = self._FetchHelper,
args = (opt,
project,
lock,
fetched,
pm,
sem,
err_event))
threads.add(t)
t.start()
for t in threads:
t.join()
# If we saw an error, exit with code 1 so that other scripts can check.
if err_event.isSet():
print >>sys.stderr, '\nerror: Exited sync due to fetch errors'
sys.exit(1)
pm.end()
for project in projects:
project.bare_git.gc('--auto')
return fetched
def _Fetch(self, projects):
fetched = set()
pm = Progress('Fetching projects', len(projects))
for project in projects:
pm.update()
if project.Sync_NetworkHalf():
fetched.add(project.gitdir)
else:
print >>sys.stderr, 'error: Cannot fetch %s' % project.name
sys.exit(1)
pm.end()
for project in projects:
project.bare_git.gc('--auto')
return fetched
def _Fetch(self, projects, opt):
fetched = set()
pm = Progress('Fetching projects', len(projects))
if self.jobs == 1:
for project in projects:
pm.update()
if project.Sync_NetworkHalf(quiet=opt.quiet):
fetched.add(project.gitdir)
else:
print >>sys.stderr, 'error: Cannot fetch %s' % project.name
if opt.force_broken:
print >>sys.stderr, 'warn: --force-broken, continuing to sync'
else:
sys.exit(1)
else:
threads = set()
lock = _threading.Lock()
sem = _threading.Semaphore(self.jobs)
err_event = _threading.Event()
for project in projects:
# Check for any errors before starting any new threads.
# ...we'll let existing threads finish, though.
if err_event.isSet():
break
sem.acquire()
t = _threading.Thread(target = self._FetchHelper,
args = (opt,
project,
lock,
fetched,
pm,
sem,
err_event))
threads.add(t)
t.start()
for t in threads:
t.join()
# If we saw an error, exit with code 1 so that other scripts can check.
if err_event.isSet():
print >>sys.stderr, '\nerror: Exited sync due to fetch errors'
sys.exit(1)
pm.end()
for project in projects:
project.bare_git.gc('--auto')
return fetched
def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
err = []
success = []
all = self.GetProjects(args[1:])
pm = Progress('Checkout %s' % nb, len(all))
for project in all:
pm.update()
status = project.CheckoutBranch(nb)
if status is not None:
if status:
success.append(project)
else:
err.append(project)
pm.end()
if err:
for p in err:
print >>sys.stderr,\
"error: %s/: cannot checkout %s" \
% (p.relpath, nb)
sys.exit(1)
elif not success or len(success) != len(all):
#branch does not exist locally, check all remotes
print >>sys.stderr, "note: no project has local branch '%s', searching remotes..." % nb
for project in all:
rbranches = project.work_git.branch("-r")
rbranches = rbranches.strip().split('\n')
rbranches = map(lambda s: s.split('->')[0].strip(), rbranches)
rbranches = [b.replace('origin/', '') for b in rbranches if b.startswith('origin/')]
if nb not in rbranches:
#remote branch doesn't exist for this repo, quit with error message
print >>sys.stderr, "error: remote branch '%s' does not exist on '%s', can't complete checkout" % (nb, project.name)
sys.exit(1)
#branch exists on all remotes, run checkout everywhere
print >>sys.stderr, "note: remote branches all exist! checking out now"
for project in all:
print >>sys.stdout, "%s:" % project.name
out = project.work_git.checkout(nb)
print >>sys.stdout, out
print >>sys.stdout
def _Fetch(self, projects, opt, err_event):
fetched = set()
lock = _threading.Lock()
pm = Progress('Fetching', len(projects))
objdir_project_map = dict()
for project in projects:
objdir_project_map.setdefault(project.objdir, []).append(project)
threads = set()
sem = _threading.Semaphore(self.jobs)
for project_list in objdir_project_map.values():
# Check for any errors before running any more tasks.
# ...we'll let existing threads finish, though.
if err_event.is_set() and opt.fail_fast:
break
sem.acquire()
kwargs = dict(opt=opt,
projects=project_list,
sem=sem,
lock=lock,
fetched=fetched,
pm=pm,
err_event=err_event,
clone_filter=self.manifest.CloneFilter)
if self.jobs > 1:
t = _threading.Thread(target=self._FetchProjectList,
kwargs=kwargs)
# Ensure that Ctrl-C will not freeze the repo process.
t.daemon = True
threads.add(t)
t.start()
else:
self._FetchProjectList(**kwargs)
for t in threads:
t.join()
pm.end()
self._fetch_times.Save()
if not self.manifest.IsArchive:
self._GCProjects(projects, opt, err_event)
return fetched
def _Checkout(self, all_projects, opt, err_results):
"""Checkout projects listed in all_projects
Args:
all_projects: List of all projects that should be checked out.
opt: Program options returned from optparse. See _Options().
err_results: A list of strings, paths to git repos where checkout failed.
"""
ret = True
# Only checkout projects with worktrees.
all_projects = [x for x in all_projects if x.worktree]
pm = Progress('Checking out', len(all_projects))
def _ProcessResults(results):
for (success, project, start, finish) in results:
self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
start, finish, success)
# Check for any errors before running any more tasks.
# ...we'll let existing threads finish, though.
if not success:
err_results.append(project.relpath)
if opt.fail_fast:
return False
pm.update(msg=project.name)
return True
# NB: Multiprocessing is heavy, so don't spin it up for one job.
if len(all_projects) == 1 or opt.jobs == 1:
if not _ProcessResults(self._CheckoutOne(opt, x) for x in all_projects):
ret = False
else:
with multiprocessing.Pool(opt.jobs) as pool:
results = pool.imap_unordered(
functools.partial(self._CheckoutOne, opt),
all_projects,
chunksize=WORKER_BATCH_SIZE)
if not _ProcessResults(results):
ret = False
pool.close()
pm.end()
return ret
def FromXml_Local_2(self, old):
shutil.rmtree(old.manifestProject.worktree)
os.remove(old._manifestFile)
my_remote = self._Remote().name
new_base = os.path.join(self.repodir, 'projects')
old_base = os.path.join(self.repodir, 'projects.old')
os.rename(new_base, old_base)
os.makedirs(new_base)
info = []
pm = Progress('Converting projects', len(self.projects))
for p in self.projects.values():
pm.update()
old_p = old.projects.get(p.name)
old_gitdir = os.path.join(old_base, '%s.git' % p.relpath)
if not os.path.isdir(old_gitdir):
continue
parent = os.path.dirname(p.gitdir)
if not os.path.isdir(parent):
os.makedirs(parent)
os.rename(old_gitdir, p.gitdir)
_rmdir(os.path.dirname(old_gitdir), self.repodir)
if not os.path.isdir(p.worktree):
os.makedirs(p.worktree)
if os.path.isdir(os.path.join(p.worktree, '.git')):
p._LinkWorkTree(relink=True)
self._CleanOldMRefs(p)
if old_p and old_p.remote.name != my_remote:
info.append("%s/: renamed remote '%s' to '%s'" \
% (p.relpath, old_p.remote.name, my_remote))
p.bare_git.remote('rename', old_p.remote.name, my_remote)
p.config.ClearCache()
self.SetMRefs(p)
pm.end()
for i in info:
print >>sys.stderr, i
def FromXml_Local_2(self, old):
shutil.rmtree(old.manifestProject.worktree)
os.remove(old._manifestFile)
my_remote = self._Remote().name
new_base = os.path.join(self.repodir, 'projects')
old_base = os.path.join(self.repodir, 'projects.old')
os.rename(new_base, old_base)
os.makedirs(new_base)
info = []
pm = Progress('Converting projects', len(self.projects))
for p in self.projects.values():
pm.update()
old_p = old.projects.get(p.name)
old_gitdir = os.path.join(old_base, '%s.git' % p.relpath)
if not os.path.isdir(old_gitdir):
continue
parent = os.path.dirname(p.gitdir)
if not os.path.isdir(parent):
os.makedirs(parent)
os.rename(old_gitdir, p.gitdir)
_rmdir(os.path.dirname(old_gitdir), self.repodir)
if not os.path.isdir(p.worktree):
os.makedirs(p.worktree)
if os.path.isdir(os.path.join(p.worktree, '.git')):
p._LinkWorkTree(relink=True)
self._CleanOldMRefs(p)
if old_p and old_p.remote.name != my_remote:
info.append("%s/: renamed remote '%s' to '%s'" \
% (p.relpath, old_p.remote.name, my_remote))
p.bare_git.remote('rename', old_p.remote.name, my_remote)
p.config.ClearCache()
self.SetMRefs(p)
pm.end()
for i in info:
print >> sys.stderr, i
def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
if not git.check_ref_format('heads/%s' % nb):
print("error: '%s' is not a valid name" % nb, file=sys.stderr)
sys.exit(1)
nb = args[0]
err = []
success = []
all_projects = self.GetProjects(args[1:])
pm = Progress('Abandon %s' % nb, len(all_projects))
for project in all_projects:
pm.update()
status = project.AbandonBranch(nb)
if status is not None:
if status:
success.append(project)
else:
err.append(project)
pm.end()
if err:
for p in err:
print("error: %s/: cannot abandon %s" % (p.relpath, nb),
file=sys.stderr)
sys.exit(1)
elif not success:
print('error: no project has branch %s' % nb, file=sys.stderr)
sys.exit(1)
else:
print('Abandoned in %d project(s):\n %s' %
(len(success), '\n '.join(p.relpath for p in success)),
file=sys.stderr)
def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
if not git.check_ref_format('heads/%s' % nb):
print("error: '%s' is not a valid name" % nb, file=sys.stderr)
sys.exit(1)
nb = args[0]
err = []
success = []
all_projects = self.GetProjects(args[1:])
pm = Progress('Abandon %s' % nb, len(all_projects))
for project in all_projects:
pm.update()
status = project.AbandonBranch(nb)
if status is not None:
if status:
success.append(project)
else:
err.append(project)
pm.end()
if err:
for p in err:
print("error: %s/: cannot abandon %s" % (p.relpath, nb),
file=sys.stderr)
sys.exit(1)
elif not success:
print('error: no project has branch %s' % nb, file=sys.stderr)
sys.exit(1)
else:
print('Abandoned in %d project(s):\n %s'
% (len(success), '\n '.join(p.relpath for p in success)),
file=sys.stderr)
def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
err = []
all = self.GetProjects(args[1:])
pm = Progress('Checkout %s' % nb, len(all))
for project in all:
pm.update()
if not project.CheckoutBranch(nb):
err.append(project)
pm.end()
if err:
if len(err) == len(all):
print >> sys.stderr, 'error: no project has branch %s' % nb
else:
for p in err:
print >>sys.stderr,\
"error: %s/: cannot checkout %s" \
% (p.relpath, nb)
sys.exit(1)
def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
err = []
all = self.GetProjects(args[1:])
pm = Progress('Checkout %s' % nb, len(all))
for project in all:
pm.update()
if not project.CheckoutBranch(nb):
err.append(project)
pm.end()
if err:
if len(err) == len(all):
print >>sys.stderr, 'error: no project has branch %s' % nb
else:
for p in err:
print >>sys.stderr,\
"error: %s/: cannot checkout %s" \
% (p.relpath, nb)
sys.exit(1)
def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
if not git.check_ref_format('heads/%s' % nb):
print("error: '%s' is not a valid name" % nb, file=sys.stderr)
sys.exit(1)
err = []
projects = []
if not opt.all:
projects = args[1:]
if len(projects) < 1:
projects = ['.',] # start it in the local project by default
all_projects = self.GetProjects(projects,
missing_ok=bool(self.gitc_manifest))
# This must happen after we find all_projects, since GetProjects may need
# the local directory, which will disappear once we save the GITC manifest.
if self.gitc_manifest:
gitc_projects = self.GetProjects(projects, manifest=self.gitc_manifest,
missing_ok=True)
for project in gitc_projects:
if project.old_revision:
project.already_synced = True
else:
project.already_synced = False
project.old_revision = project.revisionExpr
project.revisionExpr = None
# Save the GITC manifest.
gitc_utils.save_manifest(self.gitc_manifest)
# Make sure we have a valid CWD
if not os.path.exists(os.getcwd()):
os.chdir(self.manifest.topdir)
pm = Progress('Starting %s' % nb, len(all_projects))
for project in all_projects:
pm.update()
if self.gitc_manifest:
gitc_project = self.gitc_manifest.paths[project.relpath]
# Sync projects that have not been opened.
if not gitc_project.already_synced:
proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir,
project.relpath)
project.worktree = proj_localdir
if not os.path.exists(proj_localdir):
os.makedirs(proj_localdir)
project.Sync_NetworkHalf()
sync_buf = SyncBuffer(self.manifest.manifestProject.config)
project.Sync_LocalHalf(sync_buf)
project.revisionId = gitc_project.old_revision
# If the current revision is immutable, such as a SHA1, a tag or
# a change, then we can't push back to it. Substitute with
# dest_branch, if defined; or with manifest default revision instead.
branch_merge = ''
if IsImmutable(project.revisionExpr):
if project.dest_branch:
branch_merge = project.dest_branch
else:
branch_merge = self.manifest.default.revisionExpr
if not project.StartBranch(nb, branch_merge=branch_merge):
err.append(project)
pm.end()
if err:
for p in err:
print("error: %s/: cannot start %s" % (p.relpath, nb),
file=sys.stderr)
sys.exit(1)
def Execute(self, opt, args):
nb = args[0]
err = []
projects = []
if not opt.all:
projects = args[1:]
if len(projects) < 1:
projects = ['.'] # start it in the local project by default
all_projects = self.GetProjects(projects,
missing_ok=bool(self.gitc_manifest))
# This must happen after we find all_projects, since GetProjects may need
# the local directory, which will disappear once we save the GITC manifest.
if self.gitc_manifest:
gitc_projects = self.GetProjects(projects,
manifest=self.gitc_manifest,
missing_ok=True)
for project in gitc_projects:
if project.old_revision:
project.already_synced = True
else:
project.already_synced = False
project.old_revision = project.revisionExpr
project.revisionExpr = None
# Save the GITC manifest.
gitc_utils.save_manifest(self.gitc_manifest)
# Make sure we have a valid CWD
if not os.path.exists(os.getcwd()):
os.chdir(self.manifest.topdir)
pm = Progress('Syncing %s' % nb,
len(all_projects),
quiet=opt.quiet)
for project in all_projects:
gitc_project = self.gitc_manifest.paths[project.relpath]
# Sync projects that have not been opened.
if not gitc_project.already_synced:
proj_localdir = os.path.join(
self.gitc_manifest.gitc_client_dir, project.relpath)
project.worktree = proj_localdir
if not os.path.exists(proj_localdir):
os.makedirs(proj_localdir)
project.Sync_NetworkHalf()
sync_buf = SyncBuffer(self.manifest.manifestProject.config)
project.Sync_LocalHalf(sync_buf)
project.revisionId = gitc_project.old_revision
pm.update()
pm.end()
def _ProcessResults(_pool, pm, results):
for (result, project) in results:
if not result:
err.append(project)
pm.update()
self.ExecuteInParallel(opt.jobs,
functools.partial(self._ExecuteOne,
opt.revision, nb),
all_projects,
callback=_ProcessResults,
output=Progress('Starting %s' % (nb, ),
len(all_projects),
quiet=opt.quiet))
if err:
for p in err:
print("error: %s/: cannot start %s" % (p.relpath, nb),
file=sys.stderr)
sys.exit(1)
def Execute(self, opt, args):
nb = args[0]
err = []
projects = []
if not opt.all:
projects = args[1:]
if len(projects) < 1:
projects = [
'.',
] # start it in the local project by default
all_projects = self.GetProjects(projects,
missing_ok=bool(self.gitc_manifest))
# This must happen after we find all_projects, since GetProjects may need
# the local directory, which will disappear once we save the GITC manifest.
if self.gitc_manifest:
gitc_projects = self.GetProjects(projects,
manifest=self.gitc_manifest,
missing_ok=True)
for project in gitc_projects:
if project.old_revision:
project.already_synced = True
else:
project.already_synced = False
project.old_revision = project.revisionExpr
project.revisionExpr = None
# Save the GITC manifest.
gitc_utils.save_manifest(self.gitc_manifest)
# Make sure we have a valid CWD
if not os.path.exists(os.getcwd()):
os.chdir(self.manifest.topdir)
pm = Progress('Starting %s' % nb, len(all_projects))
for project in all_projects:
pm.update()
if self.gitc_manifest:
gitc_project = self.gitc_manifest.paths[project.relpath]
# Sync projects that have not been opened.
if not gitc_project.already_synced:
proj_localdir = os.path.join(
self.gitc_manifest.gitc_client_dir, project.relpath)
project.worktree = proj_localdir
if not os.path.exists(proj_localdir):
os.makedirs(proj_localdir)
project.Sync_NetworkHalf()
sync_buf = SyncBuffer(self.manifest.manifestProject.config)
project.Sync_LocalHalf(sync_buf)
project.revisionId = gitc_project.old_revision
# If the current revision is immutable, such as a SHA1, a tag or
# a change, then we can't push back to it. Substitute with
# dest_branch, if defined; or with manifest default revision instead.
branch_merge = ''
if IsImmutable(project.revisionExpr):
if project.dest_branch:
branch_merge = project.dest_branch
else:
branch_merge = self.manifest.default.revisionExpr
if not project.StartBranch(nb, branch_merge=branch_merge):
err.append(project)
pm.end()
if err:
for p in err:
print("error: %s/: cannot start %s" % (p.relpath, nb),
file=sys.stderr)
sys.exit(1)
def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
if self.jobs > 1:
soft_limit, _ = _rlimit_nofile()
self.jobs = min(self.jobs, (soft_limit - 5) / 3)
if opt.network_only and opt.detach_head:
print('error: cannot combine -n and -d', file=sys.stderr)
sys.exit(1)
if opt.network_only and opt.local_only:
print('error: cannot combine -n and -l', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_sync:
print('error: cannot combine -m and -s', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_tag:
print('error: cannot combine -m and -t', file=sys.stderr)
sys.exit(1)
if opt.manifest_server_username or opt.manifest_server_password:
if not (opt.smart_sync or opt.smart_tag):
print('error: -u and -p may only be combined with -s or -t',
file=sys.stderr)
sys.exit(1)
if None in [opt.manifest_server_username, opt.manifest_server_password]:
print('error: both -u and -p must be given', file=sys.stderr)
sys.exit(1)
if opt.manifest_name:
self.manifest.Override(opt.manifest_name)
manifest_name = opt.manifest_name
smart_sync_manifest_name = "smart_sync_override.xml"
smart_sync_manifest_path = os.path.join(
self.manifest.manifestProject.worktree, smart_sync_manifest_name)
if opt.smart_sync or opt.smart_tag:
if not self.manifest.manifest_server:
print('error: cannot smart sync: no manifest server defined in '
'manifest', file=sys.stderr)
sys.exit(1)
manifest_server = self.manifest.manifest_server
if not opt.quiet:
print('Using manifest server %s' % manifest_server)
if not '@' in manifest_server:
username = None
password = None
if opt.manifest_server_username and opt.manifest_server_password:
username = opt.manifest_server_username
password = opt.manifest_server_password
else:
try:
info = netrc.netrc()
except IOError:
# .netrc file does not exist or could not be opened
pass
else:
try:
parse_result = urllib.parse.urlparse(manifest_server)
if parse_result.hostname:
auth = info.authenticators(parse_result.hostname)
if auth:
username, _account, password = auth
else:
print('No credentials found for %s in .netrc'
% parse_result.hostname, file=sys.stderr)
except netrc.NetrcParseError as e:
print('Error parsing .netrc file: %s' % e, file=sys.stderr)
if (username and password):
manifest_server = manifest_server.replace('://', '://%s:%s@' %
(username, password),
1)
transport = PersistentTransport(manifest_server)
if manifest_server.startswith('persistent-'):
manifest_server = manifest_server[len('persistent-'):]
try:
server = xmlrpc.client.Server(manifest_server, transport=transport)
if opt.smart_sync:
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
branch = b.merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
env = os.environ.copy()
if 'SYNC_TARGET' in env:
target = env['SYNC_TARGET']
[success, manifest_str] = server.GetApprovedManifest(branch, target)
elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
target = '%s-%s' % (env['TARGET_PRODUCT'],
env['TARGET_BUILD_VARIANT'])
[success, manifest_str] = server.GetApprovedManifest(branch, target)
else:
[success, manifest_str] = server.GetApprovedManifest(branch)
else:
assert(opt.smart_tag)
[success, manifest_str] = server.GetManifest(opt.smart_tag)
if success:
manifest_name = smart_sync_manifest_name
try:
f = open(smart_sync_manifest_path, 'w')
try:
f.write(manifest_str)
finally:
f.close()
except IOError as e:
print('error: cannot write manifest to %s:\n%s'
% (smart_sync_manifest_path, e),
file=sys.stderr)
sys.exit(1)
self._ReloadManifest(manifest_name)
else:
print('error: manifest server RPC call failed: %s' %
manifest_str, file=sys.stderr)
sys.exit(1)
except (socket.error, IOError, xmlrpc.client.Fault) as e:
print('error: cannot connect to manifest server %s:\n%s'
% (self.manifest.manifest_server, e), file=sys.stderr)
sys.exit(1)
except xmlrpc.client.ProtocolError as e:
print('error: cannot connect to manifest server %s:\n%d %s'
% (self.manifest.manifest_server, e.errcode, e.errmsg),
file=sys.stderr)
sys.exit(1)
else: # Not smart sync or smart tag mode
if os.path.isfile(smart_sync_manifest_path):
try:
os.remove(smart_sync_manifest_path)
except OSError as e:
print('error: failed to remove existing smart sync override manifest: %s' %
e, file=sys.stderr)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest, quiet=opt.quiet)
if not opt.local_only:
start = time.time()
success = mp.Sync_NetworkHalf(quiet=opt.quiet,
current_branch_only=opt.current_branch_only,
no_tags=opt.no_tags,
optimized_fetch=opt.optimized_fetch,
submodules=self.manifest.HasSubmodules)
finish = time.time()
self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
start, finish, success)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
start = time.time()
mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
clean = syncbuf.Finish()
self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
start, time.time(), clean)
if not clean:
sys.exit(1)
self._ReloadManifest(manifest_name)
if opt.jobs is None:
self.jobs = self.manifest.default.sync_j
if self.gitc_manifest:
gitc_manifest_projects = self.GetProjects(args,
missing_ok=True)
gitc_projects = []
opened_projects = []
for project in gitc_manifest_projects:
if project.relpath in self.gitc_manifest.paths and \
self.gitc_manifest.paths[project.relpath].old_revision:
opened_projects.append(project.relpath)
else:
gitc_projects.append(project.relpath)
if not args:
gitc_projects = None
if gitc_projects != [] and not opt.local_only:
print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name)
if manifest_name:
manifest.Override(manifest_name)
else:
manifest.Override(self.manifest.manifestFile)
gitc_utils.generate_gitc_manifest(self.gitc_manifest,
manifest,
gitc_projects)
print('GITC client successfully synced.')
# The opened projects need to be synced as normal, therefore we
# generate a new args list to represent the opened projects.
# TODO: make this more reliable -- if there's a project name/path overlap,
# this may choose the wrong project.
args = [os.path.relpath(self.manifest.paths[path].worktree, os.getcwd())
for path in opened_projects]
if not args:
return
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
self._fetch_times = _FetchTimes(self.manifest)
if not opt.local_only:
to_fetch = []
now = time.time()
if _ONE_DAY_S <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.extend(all_projects)
to_fetch.sort(key=self._fetch_times.Get, reverse=True)
fetched = self._Fetch(to_fetch, opt)
_PostRepoFetch(rp, opt.no_repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
return
# Iteratively fetch missing and/or nested unregistered submodules
previously_missing_set = set()
while True:
self._ReloadManifest(manifest_name)
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
missing = []
for project in all_projects:
if project.gitdir not in fetched:
missing.append(project)
if not missing:
break
# Stop us from non-stopped fetching actually-missing repos: If set of
# missing repos has not been changed from last fetch, we break.
missing_set = set(p.name for p in missing)
if previously_missing_set == missing_set:
break
previously_missing_set = missing_set
fetched.update(self._Fetch(missing, opt))
if self.manifest.IsMirror or self.manifest.IsArchive:
# bail out now, we have no working tree
return
if self.UpdateProjectList():
sys.exit(1)
syncbuf = SyncBuffer(mp.config,
detach_head = opt.detach_head)
pm = Progress('Syncing work tree', len(all_projects))
for project in all_projects:
pm.update()
if project.worktree:
start = time.time()
project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
start, time.time(), syncbuf.Recently())
pm.end()
print(file=sys.stderr)
if not syncbuf.Finish():
sys.exit(1)
# If there's a notice that's supposed to print at the end of the sync, print
# it now...
if self.manifest.notice:
print(self.manifest.notice)
def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
if self.jobs > 1:
soft_limit, _ = _rlimit_nofile()
self.jobs = min(self.jobs, (soft_limit - 5) / 3)
if opt.network_only and opt.detach_head:
print >>sys.stderr, 'error: cannot combine -n and -d'
sys.exit(1)
if opt.network_only and opt.local_only:
print >>sys.stderr, 'error: cannot combine -n and -l'
sys.exit(1)
if opt.smart_sync or opt.smart_tag:
if not self.manifest.manifest_server:
print >>sys.stderr, \
'error: cannot smart sync: no manifest server defined in manifest'
sys.exit(1)
try:
server = xmlrpclib.Server(self.manifest.manifest_server)
if opt.smart_sync:
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
branch = b.merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
env = os.environ.copy()
if (env.has_key('TARGET_PRODUCT') and
env.has_key('TARGET_BUILD_VARIANT')):
target = '%s-%s' % (env['TARGET_PRODUCT'],
env['TARGET_BUILD_VARIANT'])
[success, manifest_str] = server.GetApprovedManifest(branch, target)
else:
[success, manifest_str] = server.GetApprovedManifest(branch)
else:
assert(opt.smart_tag)
[success, manifest_str] = server.GetManifest(opt.smart_tag)
if success:
manifest_name = "smart_sync_override.xml"
manifest_path = os.path.join(self.manifest.manifestProject.worktree,
manifest_name)
try:
f = open(manifest_path, 'w')
try:
f.write(manifest_str)
finally:
f.close()
except IOError:
print >>sys.stderr, 'error: cannot write manifest to %s' % \
manifest_path
sys.exit(1)
self.manifest.Override(manifest_name)
else:
print >>sys.stderr, 'error: %s' % manifest_str
sys.exit(1)
except socket.error:
print >>sys.stderr, 'error: cannot connect to manifest server %s' % (
self.manifest.manifest_server)
sys.exit(1)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest)
if not opt.local_only:
mp.Sync_NetworkHalf(quiet=opt.quiet)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
mp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1)
self.manifest._Unload()
if opt.jobs is None:
self.jobs = self.manifest.default.sync_j
all = self.GetProjects(args, missing_ok=True)
if not opt.local_only:
to_fetch = []
now = time.time()
if (24 * 60 * 60) <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.extend(all)
fetched = self._Fetch(to_fetch, opt)
_PostRepoFetch(rp, opt.no_repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
return
self.manifest._Unload()
all = self.GetProjects(args, missing_ok=True)
missing = []
for project in all:
if project.gitdir not in fetched:
missing.append(project)
self._Fetch(missing, opt)
if self.manifest.IsMirror:
# bail out now, we have no working tree
return
if self.UpdateProjectList():
sys.exit(1)
syncbuf = SyncBuffer(mp.config,
detach_head = opt.detach_head)
pm = Progress('Syncing work tree', len(all))
for project in all:
pm.update()
if project.worktree:
project.Sync_LocalHalf(syncbuf)
pm.end()
print >>sys.stderr
if not syncbuf.Finish():
sys.exit(1)
# If there's a notice that's supposed to print at the end of the sync, print
# it now...
if self.manifest.notice:
print self.manifest.notice
def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
if self.jobs > 1:
soft_limit, _ = _rlimit_nofile()
self.jobs = min(self.jobs, (soft_limit - 5) // 3)
if opt.network_only and opt.detach_head:
print('error: cannot combine -n and -d', file=sys.stderr)
sys.exit(1)
if opt.network_only and opt.local_only:
print('error: cannot combine -n and -l', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_sync:
print('error: cannot combine -m and -s', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_tag:
print('error: cannot combine -m and -t', file=sys.stderr)
sys.exit(1)
if opt.manifest_server_username or opt.manifest_server_password:
if not (opt.smart_sync or opt.smart_tag):
print('error: -u and -p may only be combined with -s or -t',
file=sys.stderr)
sys.exit(1)
if None in [
opt.manifest_server_username, opt.manifest_server_password
]:
print('error: both -u and -p must be given', file=sys.stderr)
sys.exit(1)
if opt.manifest_name:
self.manifest.Override(opt.manifest_name)
manifest_name = opt.manifest_name
smart_sync_manifest_name = "smart_sync_override.xml"
smart_sync_manifest_path = os.path.join(
self.manifest.manifestProject.worktree, smart_sync_manifest_name)
if opt.smart_sync or opt.smart_tag:
if not self.manifest.manifest_server:
print(
'error: cannot smart sync: no manifest server defined in '
'manifest',
file=sys.stderr)
sys.exit(1)
manifest_server = self.manifest.manifest_server
if not opt.quiet:
print('Using manifest server %s' % manifest_server)
if not '@' in manifest_server:
username = None
password = None
if opt.manifest_server_username and opt.manifest_server_password:
username = opt.manifest_server_username
password = opt.manifest_server_password
else:
try:
info = netrc.netrc()
except IOError:
# .netrc file does not exist or could not be opened
pass
else:
try:
parse_result = urllib.parse.urlparse(
manifest_server)
if parse_result.hostname:
auth = info.authenticators(
parse_result.hostname)
if auth:
username, _account, password = auth
else:
print(
'No credentials found for %s in .netrc'
% parse_result.hostname,
file=sys.stderr)
except netrc.NetrcParseError as e:
print('Error parsing .netrc file: %s' % e,
file=sys.stderr)
if (username and password):
manifest_server = manifest_server.replace(
'://', '://%s:%s@' % (username, password), 1)
transport = PersistentTransport(manifest_server)
if manifest_server.startswith('persistent-'):
manifest_server = manifest_server[len('persistent-'):]
try:
server = xmlrpc.client.Server(manifest_server,
transport=transport)
if opt.smart_sync:
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
branch = b.merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
env = os.environ.copy()
if 'SYNC_TARGET' in env:
target = env['SYNC_TARGET']
[success, manifest_str
] = server.GetApprovedManifest(branch, target)
elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
target = '%s-%s' % (env['TARGET_PRODUCT'],
env['TARGET_BUILD_VARIANT'])
[success, manifest_str
] = server.GetApprovedManifest(branch, target)
else:
[success,
manifest_str] = server.GetApprovedManifest(branch)
else:
assert (opt.smart_tag)
[success, manifest_str] = server.GetManifest(opt.smart_tag)
if success:
manifest_name = smart_sync_manifest_name
try:
f = open(smart_sync_manifest_path, 'w')
try:
f.write(manifest_str)
finally:
f.close()
except IOError as e:
print('error: cannot write manifest to %s:\n%s' %
(smart_sync_manifest_path, e),
file=sys.stderr)
sys.exit(1)
self._ReloadManifest(manifest_name)
else:
print('error: manifest server RPC call failed: %s' %
manifest_str,
file=sys.stderr)
sys.exit(1)
except (socket.error, IOError, xmlrpc.client.Fault) as e:
print('error: cannot connect to manifest server %s:\n%s' %
(self.manifest.manifest_server, e),
file=sys.stderr)
sys.exit(1)
except xmlrpc.client.ProtocolError as e:
print('error: cannot connect to manifest server %s:\n%d %s' %
(self.manifest.manifest_server, e.errcode, e.errmsg),
file=sys.stderr)
sys.exit(1)
else: # Not smart sync or smart tag mode
if os.path.isfile(smart_sync_manifest_path):
try:
platform_utils.remove(smart_sync_manifest_path)
except OSError as e:
print(
'error: failed to remove existing smart sync override manifest: %s'
% e,
file=sys.stderr)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest, quiet=opt.quiet)
if not opt.local_only:
start = time.time()
success = mp.Sync_NetworkHalf(
quiet=opt.quiet,
current_branch_only=opt.current_branch_only,
no_tags=opt.no_tags,
optimized_fetch=opt.optimized_fetch,
submodules=self.manifest.HasSubmodules)
finish = time.time()
self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK, start,
finish, success)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
start = time.time()
mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
clean = syncbuf.Finish()
self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL, start,
time.time(), clean)
if not clean:
sys.exit(1)
self._ReloadManifest(manifest_name)
if opt.jobs is None:
self.jobs = self.manifest.default.sync_j
if self.gitc_manifest:
gitc_manifest_projects = self.GetProjects(args, missing_ok=True)
gitc_projects = []
opened_projects = []
for project in gitc_manifest_projects:
if project.relpath in self.gitc_manifest.paths and \
self.gitc_manifest.paths[project.relpath].old_revision:
opened_projects.append(project.relpath)
else:
gitc_projects.append(project.relpath)
if not args:
gitc_projects = None
if gitc_projects != [] and not opt.local_only:
print('Updating GITC client: %s' %
self.gitc_manifest.gitc_client_name)
manifest = GitcManifest(self.repodir,
self.gitc_manifest.gitc_client_name)
if manifest_name:
manifest.Override(manifest_name)
else:
manifest.Override(self.manifest.manifestFile)
gitc_utils.generate_gitc_manifest(self.gitc_manifest, manifest,
gitc_projects)
print('GITC client successfully synced.')
# The opened projects need to be synced as normal, therefore we
# generate a new args list to represent the opened projects.
# TODO: make this more reliable -- if there's a project name/path overlap,
# this may choose the wrong project.
args = [
os.path.relpath(self.manifest.paths[path].worktree,
os.getcwd()) for path in opened_projects
]
if not args:
return
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
self._fetch_times = _FetchTimes(self.manifest)
if not opt.local_only:
to_fetch = []
now = time.time()
if _ONE_DAY_S <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.extend(all_projects)
to_fetch.sort(key=self._fetch_times.Get, reverse=True)
fetched = self._Fetch(to_fetch, opt)
_PostRepoFetch(rp, opt.no_repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
return
# Iteratively fetch missing and/or nested unregistered submodules
previously_missing_set = set()
while True:
self._ReloadManifest(manifest_name)
all_projects = self.GetProjects(
args, missing_ok=True, submodules_ok=opt.fetch_submodules)
missing = []
for project in all_projects:
if project.gitdir not in fetched:
missing.append(project)
if not missing:
break
# Stop us from non-stopped fetching actually-missing repos: If set of
# missing repos has not been changed from last fetch, we break.
missing_set = set(p.name for p in missing)
if previously_missing_set == missing_set:
break
previously_missing_set = missing_set
fetched.update(self._Fetch(missing, opt))
if self.manifest.IsMirror or self.manifest.IsArchive:
# bail out now, we have no working tree
return
if self.UpdateProjectList(opt):
sys.exit(1)
syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head)
pm = Progress('Syncing work tree', len(all_projects))
for project in all_projects:
pm.update()
if project.worktree:
start = time.time()
project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
start, time.time(), syncbuf.Recently())
pm.end()
print(file=sys.stderr)
if not syncbuf.Finish():
sys.exit(1)
# If there's a notice that's supposed to print at the end of the sync, print
# it now...
if self.manifest.notice:
print(self.manifest.notice)
def Execute(self, opt, args):
if opt.jobs:
self.jobs = 3
if opt.network_only and opt.detach_head:
print >> sys.stderr, 'error: cannot combine -n and -d'
sys.exit(1)
if opt.network_only and opt.local_only:
print >> sys.stderr, 'error: cannot combine -n and -l'
sys.exit(1)
if opt.smart_sync:
if not self.manifest.manifest_server:
print >>sys.stderr, \
'error: cannot smart sync: no manifest server defined in manifest'
sys.exit(1)
try:
server = xmlrpclib.Server(self.manifest.manifest_server)
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
branch = b.merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
env = os.environ.copy()
if (env.has_key('TARGET_PRODUCT')
and env.has_key('TARGET_BUILD_VARIANT')):
target = '%s-%s' % (env['TARGET_PRODUCT'],
env['TARGET_BUILD_VARIANT'])
[success, manifest_str
] = server.GetApprovedManifest(branch, target)
else:
[success,
manifest_str] = server.GetApprovedManifest(branch)
if success:
manifest_name = "smart_sync_override.xml"
manifest_path = os.path.join(
self.manifest.manifestProject.worktree, manifest_name)
try:
f = open(manifest_path, 'w')
try:
f.write(manifest_str)
finally:
f.close()
except IOError:
print >>sys.stderr, 'error: cannot write manifest to %s' % \
manifest_path
sys.exit(1)
self.manifest.Override(manifest_name)
else:
print >> sys.stderr, 'error: %s' % manifest_str
sys.exit(1)
except socket.error:
print >> sys.stderr, 'error: cannot connect to manifest server %s' % (
self.manifest.manifest_server)
sys.exit(1)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest)
if not opt.local_only:
mp.Sync_NetworkHalf(quiet=opt.quiet)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
mp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1)
self.manifest._Unload()
all = self.GetProjects(args, missing_ok=True)
if not opt.local_only:
to_fetch = []
now = time.time()
if (24 * 60 * 60) <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.extend(all)
fetched = self._Fetch(to_fetch, opt)
_PostRepoFetch(rp, True)
if opt.network_only:
# bail out now; the rest touches the working tree
return
self.manifest._Unload()
all = self.GetProjects(args, missing_ok=True)
missing = []
for project in all:
if project.gitdir not in fetched:
missing.append(project)
self._Fetch(missing, opt)
if self.manifest.IsMirror:
# bail out now, we have no working tree
return
if self.UpdateProjectList():
sys.exit(1)
syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head)
pm = Progress('Syncing work tree', len(all))
for project in all:
pm.update()
if project.worktree:
project.Sync_LocalHalf(syncbuf)
pm.end()
print >> sys.stderr
if not syncbuf.Finish():
sys.exit(1)
# If there's a notice that's supposed to print at the end of the sync, print
# it now...
if self.manifest.notice:
print self.manifest.notice
def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
if opt.network_only and opt.detach_head:
print >> sys.stderr, "error: cannot combine -n and -d"
sys.exit(1)
if opt.network_only and opt.local_only:
print >> sys.stderr, "error: cannot combine -n and -l"
sys.exit(1)
if opt.smart_sync:
if not self.manifest.manifest_server:
print >> sys.stderr, "error: cannot smart sync: no manifest server defined in manifest"
sys.exit(1)
try:
server = xmlrpclib.Server(self.manifest.manifest_server)
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
branch = b.merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS) :]
env = os.environ.copy()
if env.has_key("TARGET_PRODUCT") and env.has_key("TARGET_BUILD_VARIANT"):
target = "%s-%s" % (env["TARGET_PRODUCT"], env["TARGET_BUILD_VARIANT"])
[success, manifest_str] = server.GetApprovedManifest(branch, target)
else:
[success, manifest_str] = server.GetApprovedManifest(branch)
if success:
manifest_name = "smart_sync_override.xml"
manifest_path = os.path.join(self.manifest.manifestProject.worktree, manifest_name)
try:
f = open(manifest_path, "w")
try:
f.write(manifest_str)
finally:
f.close()
except IOError:
print >> sys.stderr, "error: cannot write manifest to %s" % manifest_path
sys.exit(1)
self.manifest.Override(manifest_name)
else:
print >> sys.stderr, "error: %s" % manifest_str
sys.exit(1)
except socket.error:
print >> sys.stderr, "error: cannot connect to manifest server %s" % (self.manifest.manifest_server)
sys.exit(1)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest)
if not opt.local_only:
mp.Sync_NetworkHalf(quiet=opt.quiet)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
mp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1)
self.manifest._Unload()
all = self.GetProjects(args, missing_ok=True)
if not opt.local_only:
to_fetch = []
now = time.time()
if (24 * 60 * 60) <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.extend(all)
fetched = self._Fetch(to_fetch, opt)
_PostRepoFetch(rp, opt.no_repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
return
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
mp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1)
_ReloadManifest(self)
mp = self.manifest.manifestProject
all = self.GetProjects(args, missing_ok=True)
missing = []
for project in all:
if project.gitdir not in fetched:
missing.append(project)
self._Fetch(missing, opt)
if self.manifest.IsMirror:
# bail out now, we have no working tree
return
if self.UpdateProjectList():
sys.exit(1)
syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head)
pm = Progress("Syncing work tree", len(all))
for project in all:
pm.update()
if project.worktree:
project.Sync_LocalHalf(syncbuf)
pm.end()
print >> sys.stderr
if not syncbuf.Finish():
sys.exit(1)
def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
if self.jobs > 1:
soft_limit, _ = _rlimit_nofile()
self.jobs = min(self.jobs, (soft_limit - 5) / 3)
if opt.network_only and opt.detach_head:
print('error: cannot combine -n and -d', file=sys.stderr)
sys.exit(1)
if opt.network_only and opt.local_only:
print('error: cannot combine -n and -l', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_sync:
print('error: cannot combine -m and -s', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_tag:
print('error: cannot combine -m and -t', file=sys.stderr)
sys.exit(1)
if opt.manifest_server_username or opt.manifest_server_password:
if not (opt.smart_sync or opt.smart_tag):
print('error: -u and -p may only be combined with -s or -t',
file=sys.stderr)
sys.exit(1)
if None in [opt.manifest_server_username, opt.manifest_server_password]:
print('error: both -u and -p must be given', file=sys.stderr)
sys.exit(1)
if opt.manifest_name:
self.manifest.Override(opt.manifest_name)
if opt.smart_sync or opt.smart_tag:
if not self.manifest.manifest_server:
print('error: cannot smart sync: no manifest server defined in'
'manifest', file=sys.stderr)
sys.exit(1)
manifest_server = self.manifest.manifest_server
if not '@' in manifest_server:
username = None
password = None
if opt.manifest_server_username and opt.manifest_server_password:
username = opt.manifest_server_username
password = opt.manifest_server_password
else:
try:
info = netrc.netrc()
except IOError:
print('.netrc file does not exist or could not be opened',
file=sys.stderr)
else:
try:
parse_result = urlparse.urlparse(manifest_server)
if parse_result.hostname:
username, _account, password = \
info.authenticators(parse_result.hostname)
except TypeError:
# TypeError is raised when the given hostname is not present
# in the .netrc file.
print('No credentials found for %s in .netrc'
% parse_result.hostname, file=sys.stderr)
except netrc.NetrcParseError as e:
print('Error parsing .netrc file: %s' % e, file=sys.stderr)
if (username and password):
manifest_server = manifest_server.replace('://', '://%s:%s@' %
(username, password),
1)
try:
server = xmlrpclib.Server(manifest_server)
if opt.smart_sync:
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
branch = b.merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
env = os.environ.copy()
if (env.has_key('TARGET_PRODUCT') and
env.has_key('TARGET_BUILD_VARIANT')):
target = '%s-%s' % (env['TARGET_PRODUCT'],
env['TARGET_BUILD_VARIANT'])
[success, manifest_str] = server.GetApprovedManifest(branch, target)
else:
[success, manifest_str] = server.GetApprovedManifest(branch)
else:
assert(opt.smart_tag)
[success, manifest_str] = server.GetManifest(opt.smart_tag)
if success:
manifest_name = "smart_sync_override.xml"
manifest_path = os.path.join(self.manifest.manifestProject.worktree,
manifest_name)
try:
f = open(manifest_path, 'w')
try:
f.write(manifest_str)
finally:
f.close()
except IOError:
print('error: cannot write manifest to %s' % manifest_path,
file=sys.stderr)
sys.exit(1)
self.manifest.Override(manifest_name)
else:
print('error: %s' % manifest_str, file=sys.stderr)
sys.exit(1)
except (socket.error, IOError, xmlrpclib.Fault) as e:
print('error: cannot connect to manifest server %s:\n%s'
% (self.manifest.manifest_server, e), file=sys.stderr)
sys.exit(1)
except xmlrpclib.ProtocolError as e:
print('error: cannot connect to manifest server %s:\n%d %s'
% (self.manifest.manifest_server, e.errcode, e.errmsg),
file=sys.stderr)
sys.exit(1)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest, quiet=opt.quiet)
if not opt.local_only:
mp.Sync_NetworkHalf(quiet=opt.quiet,
current_branch_only=opt.current_branch_only)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
mp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1)
self.manifest._Unload()
if opt.jobs is None:
self.jobs = self.manifest.default.sync_j
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
self._fetch_times = _FetchTimes(self.manifest)
if not opt.local_only:
to_fetch = []
now = time.time()
if _ONE_DAY_S <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.extend(all_projects)
to_fetch.sort(key=self._fetch_times.Get, reverse=True)
fetched = self._Fetch(to_fetch, opt)
_PostRepoFetch(rp, opt.no_repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
return
# Iteratively fetch missing and/or nested unregistered submodules
previously_missing_set = set()
while True:
self.manifest._Unload()
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
missing = []
for project in all_projects:
if project.gitdir not in fetched:
missing.append(project)
if not missing:
break
# Stop us from non-stopped fetching actually-missing repos: If set of
# missing repos has not been changed from last fetch, we break.
missing_set = set(p.name for p in missing)
if previously_missing_set == missing_set:
break
previously_missing_set = missing_set
fetched.update(self._Fetch(missing, opt))
if self.manifest.IsMirror:
# bail out now, we have no working tree
return
if self.UpdateProjectList():
sys.exit(1)
syncbuf = SyncBuffer(mp.config,
detach_head = opt.detach_head)
pm = Progress('Syncing work tree', len(all_projects))
for project in all_projects:
pm.update()
if project.worktree:
project.Sync_LocalHalf(syncbuf)
pm.end()
print(file=sys.stderr)
if not syncbuf.Finish():
sys.exit(1)
# If there's a notice that's supposed to print at the end of the sync, print
# it now...
if self.manifest.notice:
print(self.manifest.notice)
def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
if self.jobs > 1:
soft_limit, _ = _rlimit_nofile()
self.jobs = min(self.jobs, (soft_limit - 5) / 3)
if opt.network_only and opt.detach_head:
print('error: cannot combine -n and -d', file=sys.stderr)
sys.exit(1)
if opt.network_only and opt.local_only:
print('error: cannot combine -n and -l', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_sync:
print('error: cannot combine -m and -s', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_tag:
print('error: cannot combine -m and -t', file=sys.stderr)
sys.exit(1)
if opt.manifest_server_username or opt.manifest_server_password:
if not (opt.smart_sync or opt.smart_tag):
print('error: -u and -p may only be combined with -s or -t',
file=sys.stderr)
sys.exit(1)
if None in [
opt.manifest_server_username, opt.manifest_server_password
]:
print('error: both -u and -p must be given', file=sys.stderr)
sys.exit(1)
cwd = os.getcwd()
if cwd.startswith(gitc_utils.GITC_MANIFEST_DIR) and not opt.force_gitc:
print(
'WARNING this will pull all the sources like a normal repo sync.\n'
'\nIf you want to update your GITC Client View please rerun this '
'command in \n%s%s.\nOr if you actually want to pull the sources, '
'rerun with --force-gitc.' %
(gitc_utils.GITC_FS_ROOT_DIR,
cwd.split(gitc_utils.GITC_MANIFEST_DIR)[1]))
sys.exit(1)
self._gitc_sync = False
if cwd.startswith(gitc_utils.GITC_FS_ROOT_DIR):
self._gitc_sync = True
self._client_name = cwd.split(
gitc_utils.GITC_FS_ROOT_DIR)[1].split('/')[0]
self._client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR,
self._client_name)
print('Updating GITC client: %s' % self._client_name)
if opt.manifest_name:
self.manifest.Override(opt.manifest_name)
manifest_name = opt.manifest_name
smart_sync_manifest_name = "smart_sync_override.xml"
smart_sync_manifest_path = os.path.join(
self.manifest.manifestProject.worktree, smart_sync_manifest_name)
if opt.smart_sync or opt.smart_tag:
if not self.manifest.manifest_server:
print(
'error: cannot smart sync: no manifest server defined in '
'manifest',
file=sys.stderr)
sys.exit(1)
manifest_server = self.manifest.manifest_server
if not opt.quiet:
print('Using manifest server %s' % manifest_server)
if not '@' in manifest_server:
username = None
password = None
if opt.manifest_server_username and opt.manifest_server_password:
username = opt.manifest_server_username
password = opt.manifest_server_password
else:
try:
info = netrc.netrc()
except IOError:
# .netrc file does not exist or could not be opened
pass
else:
try:
parse_result = urllib.parse.urlparse(
manifest_server)
if parse_result.hostname:
auth = info.authenticators(
parse_result.hostname)
if auth:
username, _account, password = auth
else:
print(
'No credentials found for %s in .netrc'
% parse_result.hostname,
file=sys.stderr)
except netrc.NetrcParseError as e:
print('Error parsing .netrc file: %s' % e,
file=sys.stderr)
if (username and password):
manifest_server = manifest_server.replace(
'://', '://%s:%s@' % (username, password), 1)
transport = PersistentTransport(manifest_server)
if manifest_server.startswith('persistent-'):
manifest_server = manifest_server[len('persistent-'):]
try:
server = xmlrpc.client.Server(manifest_server,
transport=transport)
if opt.smart_sync:
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
branch = b.merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
env = os.environ.copy()
if 'SYNC_TARGET' in env:
target = env['SYNC_TARGET']
[success, manifest_str
] = server.GetApprovedManifest(branch, target)
elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
target = '%s-%s' % (env['TARGET_PRODUCT'],
env['TARGET_BUILD_VARIANT'])
[success, manifest_str
] = server.GetApprovedManifest(branch, target)
else:
[success,
manifest_str] = server.GetApprovedManifest(branch)
else:
assert (opt.smart_tag)
[success, manifest_str] = server.GetManifest(opt.smart_tag)
if success:
manifest_name = smart_sync_manifest_name
try:
f = open(smart_sync_manifest_path, 'w')
try:
f.write(manifest_str)
finally:
f.close()
except IOError as e:
print('error: cannot write manifest to %s:\n%s' %
(smart_sync_manifest_path, e),
file=sys.stderr)
sys.exit(1)
self._ReloadManifest(manifest_name)
else:
print('error: manifest server RPC call failed: %s' %
manifest_str,
file=sys.stderr)
sys.exit(1)
except (socket.error, IOError, xmlrpc.client.Fault) as e:
print('error: cannot connect to manifest server %s:\n%s' %
(self.manifest.manifest_server, e),
file=sys.stderr)
sys.exit(1)
except xmlrpc.client.ProtocolError as e:
print('error: cannot connect to manifest server %s:\n%d %s' %
(self.manifest.manifest_server, e.errcode, e.errmsg),
file=sys.stderr)
sys.exit(1)
else: # Not smart sync or smart tag mode
if os.path.isfile(smart_sync_manifest_path):
try:
os.remove(smart_sync_manifest_path)
except OSError as e:
print(
'error: failed to remove existing smart sync override manifest: %s'
% e,
file=sys.stderr)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest, quiet=opt.quiet)
if self._gitc_sync:
gitc_utils.generate_gitc_manifest(self._client_dir, self.manifest)
print('GITC client successfully synced.')
return
if not opt.local_only:
mp.Sync_NetworkHalf(quiet=opt.quiet,
current_branch_only=opt.current_branch_only,
no_tags=opt.no_tags,
optimized_fetch=opt.optimized_fetch)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
mp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1)
self._ReloadManifest(manifest_name)
if opt.jobs is None:
self.jobs = self.manifest.default.sync_j
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
self._fetch_times = _FetchTimes(self.manifest)
if not opt.local_only:
to_fetch = []
now = time.time()
if _ONE_DAY_S <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.extend(all_projects)
to_fetch.sort(key=self._fetch_times.Get, reverse=True)
fetched = self._Fetch(to_fetch, opt)
_PostRepoFetch(rp, opt.no_repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
return
# Iteratively fetch missing and/or nested unregistered submodules
previously_missing_set = set()
while True:
self._ReloadManifest(manifest_name)
all_projects = self.GetProjects(
args, missing_ok=True, submodules_ok=opt.fetch_submodules)
missing = []
for project in all_projects:
if project.gitdir not in fetched:
missing.append(project)
if not missing:
break
# Stop us from non-stopped fetching actually-missing repos: If set of
# missing repos has not been changed from last fetch, we break.
missing_set = set(p.name for p in missing)
if previously_missing_set == missing_set:
break
previously_missing_set = missing_set
fetched.update(self._Fetch(missing, opt))
if self.manifest.IsMirror or self.manifest.IsArchive:
# bail out now, we have no working tree
return
if self.UpdateProjectList():
sys.exit(1)
syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head)
pm = Progress('Syncing work tree', len(all_projects))
for project in all_projects:
pm.update()
if project.worktree:
project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
pm.end()
print(file=sys.stderr)
if not syncbuf.Finish():
sys.exit(1)
# If there's a notice that's supposed to print at the end of the sync, print
# it now...
if self.manifest.notice:
print(self.manifest.notice)
def Execute(self, opt, args):
if opt.network_only and opt.detach_head:
print >> sys.stderr, 'error: cannot combine -n and -d'
sys.exit(1)
if opt.network_only and opt.local_only:
print >> sys.stderr, 'error: cannot combine -n and -l'
sys.exit(1)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest)
all = self.GetProjects(args, missing_ok=True)
if not opt.local_only:
to_fetch = []
now = time.time()
if (24 * 60 * 60) <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.append(mp)
to_fetch.extend(all)
fetched = self._Fetch(to_fetch)
_PostRepoFetch(rp, opt.no_repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
return
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
mp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1)
self.manifest._Unload()
all = self.GetProjects(args, missing_ok=True)
missing = []
for project in all:
if project.gitdir not in fetched:
missing.append(project)
self._Fetch(missing)
if self.manifest.IsMirror:
# bail out now, we have no working tree
return
if self.UpdateProjectList():
sys.exit(1)
syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head)
pm = Progress('Syncing work tree', len(all))
for project in all:
pm.update()
if project.worktree:
project.Sync_LocalHalf(syncbuf)
pm.end()
print >> sys.stderr
if not syncbuf.Finish():
sys.exit(1)
def Execute(self, opt, args):
if not args:
self.Usage()
nb = args[0]
if not git.check_ref_format('heads/%s' % nb):
print("error: '%s' is not a valid name" % nb, file=sys.stderr)
sys.exit(1)
err = []
projects = []
if not opt.all:
projects = args[1:]
if len(projects) < 1:
print("error: at least one project must be specified", file=sys.stderr)
sys.exit(1)
proj_name_to_gitc_proj_dict = {}
if self.gitc_manifest:
all_projects = self.GetProjects(projects, manifest=self.gitc_manifest,
missing_ok=True)
for project in all_projects:
if project.old_revision:
project.already_synced = True
else:
project.already_synced = False
project.old_revision = project.revisionExpr
proj_name_to_gitc_proj_dict[project.name] = project
project.revisionExpr = None
# Save the GITC manifest.
gitc_utils.save_manifest(self.gitc_manifest)
all_projects = self.GetProjects(projects,
missing_ok=bool(self.gitc_manifest))
pm = Progress('Starting %s' % nb, len(all_projects))
for project in all_projects:
pm.update()
if self.gitc_manifest:
gitc_project = proj_name_to_gitc_proj_dict[project.name]
# Sync projects that have already been opened.
if not gitc_project.already_synced:
proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir,
project.relpath)
project.worktree = proj_localdir
if not os.path.exists(proj_localdir):
os.makedirs(proj_localdir)
project.Sync_NetworkHalf()
sync_buf = SyncBuffer(self.manifest.manifestProject.config)
project.Sync_LocalHalf(sync_buf)
project.revisionExpr = gitc_project.old_revision
# If the current revision is a specific SHA1 then we can't push back
# to it; so substitute with dest_branch if defined, or with manifest
# default revision instead.
branch_merge = ''
if IsId(project.revisionExpr):
if project.dest_branch:
branch_merge = project.dest_branch
else:
branch_merge = self.manifest.default.revisionExpr
if not project.StartBranch(nb, branch_merge=branch_merge):
err.append(project)
pm.end()
if err:
for p in err:
print("error: %s/: cannot start %s" % (p.relpath, nb),
file=sys.stderr)
sys.exit(1)
def Execute(self, opt, args):
if not opt.all and not args:
self.Usage()
if not opt.all:
nb = args[0]
if not git.check_ref_format('heads/%s' % nb):
print("error: '%s' is not a valid name" % nb, file=sys.stderr)
sys.exit(1)
else:
args.insert(0, None)
nb = "'All local branches'"
err = defaultdict(list)
success = defaultdict(list)
all_projects = self.GetProjects(args[1:])
pm = Progress('Abandon %s' % nb, len(all_projects))
for project in all_projects:
pm.update()
if opt.all:
branches = project.GetBranches().keys()
else:
branches = [nb]
for name in branches:
status = project.AbandonBranch(name)
if status is not None:
if status:
success[name].append(project)
else:
err[name].append(project)
pm.end()
width = 25
for name in branches:
if width < len(name):
width = len(name)
if err:
for br in err.keys():
err_msg = "error: cannot abandon %s" % br
print(err_msg, file=sys.stderr)
for proj in err[br]:
print(' ' * len(err_msg) + " | %s" % proj.relpath,
file=sys.stderr)
sys.exit(1)
elif not success:
print('error: no project has local branch(es) : %s' % nb,
file=sys.stderr)
sys.exit(1)
else:
print('Abandoned branches:', file=sys.stderr)
for br in success.keys():
if len(all_projects) > 1 and len(all_projects) == len(
success[br]):
result = "all project"
else:
result = "%s" % (
('\n' + ' ' * width + '| ').join(p.relpath
for p in success[br]))
print("%s%s| %s\n" % (br, ' ' * (width - len(br)), result),
file=sys.stderr)
def Execute(self, opt, args):
nb = args[0]
err = []
projects = []
if not opt.all:
projects = args[1:]
if len(projects) < 1:
projects = ['.'] # start it in the local project by default
all_projects = self.GetProjects(projects,
missing_ok=bool(self.gitc_manifest))
# This must happen after we find all_projects, since GetProjects may need
# the local directory, which will disappear once we save the GITC manifest.
if self.gitc_manifest:
gitc_projects = self.GetProjects(projects,
manifest=self.gitc_manifest,
missing_ok=True)
for project in gitc_projects:
if project.old_revision:
project.already_synced = True
else:
project.already_synced = False
project.old_revision = project.revisionExpr
project.revisionExpr = None
# Save the GITC manifest.
gitc_utils.save_manifest(self.gitc_manifest)
# Make sure we have a valid CWD
if not os.path.exists(os.getcwd()):
os.chdir(self.manifest.topdir)
pm = Progress('Syncing %s' % nb, len(all_projects))
for project in all_projects:
gitc_project = self.gitc_manifest.paths[project.relpath]
# Sync projects that have not been opened.
if not gitc_project.already_synced:
proj_localdir = os.path.join(
self.gitc_manifest.gitc_client_dir, project.relpath)
project.worktree = proj_localdir
if not os.path.exists(proj_localdir):
os.makedirs(proj_localdir)
project.Sync_NetworkHalf()
sync_buf = SyncBuffer(self.manifest.manifestProject.config)
project.Sync_LocalHalf(sync_buf)
project.revisionId = gitc_project.old_revision
pm.update()
pm.end()
def _ProcessResults(results):
for (result, project) in results:
if not result:
err.append(project)
pm.update()
pm = Progress('Starting %s' % nb, len(all_projects))
# NB: Multiprocessing is heavy, so don't spin it up for one job.
if len(all_projects) == 1 or opt.jobs == 1:
_ProcessResults(self._ExecuteOne(opt, nb, x) for x in all_projects)
else:
with multiprocessing.Pool(opt.jobs) as pool:
results = pool.imap_unordered(functools.partial(
self._ExecuteOne, opt, nb),
all_projects,
chunksize=WORKER_BATCH_SIZE)
_ProcessResults(results)
pm.end()
if err:
for p in err:
print("error: %s/: cannot start %s" % (p.relpath, nb),
file=sys.stderr)
sys.exit(1)
def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
if self.jobs > 1:
soft_limit, _ = _rlimit_nofile()
self.jobs = min(self.jobs, (soft_limit - 5) / 3)
if opt.network_only and opt.detach_head:
print('error: cannot combine -n and -d', file=sys.stderr)
sys.exit(1)
if opt.network_only and opt.local_only:
print('error: cannot combine -n and -l', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_sync:
print('error: cannot combine -m and -s', file=sys.stderr)
sys.exit(1)
if opt.manifest_name and opt.smart_tag:
print('error: cannot combine -m and -t', file=sys.stderr)
sys.exit(1)
if opt.manifest_server_username or opt.manifest_server_password:
if not (opt.smart_sync or opt.smart_tag):
print('error: -u and -p may only be combined with -s or -t',
file=sys.stderr)
sys.exit(1)
if None in [
opt.manifest_server_username, opt.manifest_server_password
]:
print('error: both -u and -p must be given', file=sys.stderr)
sys.exit(1)
if opt.manifest_name:
self.manifest.Override(opt.manifest_name)
manifest_name = opt.manifest_name
if opt.smart_sync or opt.smart_tag:
if not self.manifest.manifest_server:
print(
'error: cannot smart sync: no manifest server defined in '
'manifest',
file=sys.stderr)
sys.exit(1)
manifest_server = self.manifest.manifest_server
if not opt.quiet:
print('Using manifest server %s' % manifest_server)
if not '@' in manifest_server:
username = None
password = None
if opt.manifest_server_username and opt.manifest_server_password:
username = opt.manifest_server_username
password = opt.manifest_server_password
else:
try:
info = netrc.netrc()
except IOError:
print(
'.netrc file does not exist or could not be opened',
file=sys.stderr)
else:
try:
parse_result = urllib.parse.urlparse(
manifest_server)
if parse_result.hostname:
username, _account, password = \
info.authenticators(parse_result.hostname)
except TypeError:
# TypeError is raised when the given hostname is not present
# in the .netrc file.
print('No credentials found for %s in .netrc' %
parse_result.hostname,
file=sys.stderr)
except netrc.NetrcParseError as e:
print('Error parsing .netrc file: %s' % e,
file=sys.stderr)
if (username and password):
manifest_server = manifest_server.replace(
'://', '://%s:%s@' % (username, password), 1)
try:
server = xmlrpc.client.Server(manifest_server)
if opt.smart_sync:
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
branch = b.merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
env = os.environ.copy()
if 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
target = '%s-%s' % (env['TARGET_PRODUCT'],
env['TARGET_BUILD_VARIANT'])
[success, manifest_str
] = server.GetApprovedManifest(branch, target)
else:
[success,
manifest_str] = server.GetApprovedManifest(branch)
else:
assert (opt.smart_tag)
[success, manifest_str] = server.GetManifest(opt.smart_tag)
if success:
manifest_name = "smart_sync_override.xml"
manifest_path = os.path.join(
self.manifest.manifestProject.worktree, manifest_name)
try:
f = open(manifest_path, 'w')
try:
f.write(manifest_str)
finally:
f.close()
except IOError:
print('error: cannot write manifest to %s' %
manifest_path,
file=sys.stderr)
sys.exit(1)
self._ReloadManifest(manifest_name)
else:
print('error: manifest server RPC call failed: %s' %
manifest_str,
file=sys.stderr)
sys.exit(1)
except (socket.error, IOError, xmlrpc.client.Fault) as e:
print('error: cannot connect to manifest server %s:\n%s' %
(self.manifest.manifest_server, e),
file=sys.stderr)
sys.exit(1)
except xmlrpc.client.ProtocolError as e:
print('error: cannot connect to manifest server %s:\n%d %s' %
(self.manifest.manifest_server, e.errcode, e.errmsg),
file=sys.stderr)
sys.exit(1)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest, quiet=opt.quiet)
if not opt.local_only:
mp.Sync_NetworkHalf(quiet=opt.quiet,
current_branch_only=opt.current_branch_only,
no_tags=opt.no_tags)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
mp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1)
self._ReloadManifest(manifest_name)
if opt.jobs is None:
self.jobs = self.manifest.default.sync_j
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
self._fetch_times = _FetchTimes(self.manifest)
if not opt.local_only:
to_fetch = []
now = time.time()
if _ONE_DAY_S <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.extend(all_projects)
to_fetch.sort(key=self._fetch_times.Get, reverse=True)
fetched = self._Fetch(to_fetch, opt)
_PostRepoFetch(rp, opt.no_repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
return
# Iteratively fetch missing and/or nested unregistered submodules
previously_missing_set = set()
while True:
self._ReloadManifest(manifest_name)
all_projects = self.GetProjects(
args, missing_ok=True, submodules_ok=opt.fetch_submodules)
missing = []
for project in all_projects:
if project.gitdir not in fetched:
missing.append(project)
if not missing:
break
# Stop us from non-stopped fetching actually-missing repos: If set of
# missing repos has not been changed from last fetch, we break.
missing_set = set(p.name for p in missing)
if previously_missing_set == missing_set:
break
previously_missing_set = missing_set
fetched.update(self._Fetch(missing, opt))
if self.manifest.IsMirror or self.manifest.IsArchive:
# bail out now, we have no working tree
return
if self.UpdateProjectList():
sys.exit(1)
syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head)
pm = Progress('Syncing work tree', len(all_projects))
for project in all_projects:
pm.update()
if project.worktree:
project.Sync_LocalHalf(syncbuf)
pm.end()
print(file=sys.stderr)
if not syncbuf.Finish():
sys.exit(1)
# If there's a notice that's supposed to print at the end of the sync, print
# it now...
if self.manifest.notice:
print(self.manifest.notice)
def Execute(self, opt, args):
if opt.network_only and opt.detach_head:
print >>sys.stderr, 'error: cannot combine -n and -d'
sys.exit(1)
if opt.network_only and opt.local_only:
print >>sys.stderr, 'error: cannot combine -n and -l'
sys.exit(1)
rp = self.manifest.repoProject
rp.PreSync()
mp = self.manifest.manifestProject
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest)
all = self.GetProjects(args, missing_ok=True)
if not opt.local_only:
to_fetch = []
now = time.time()
if (24 * 60 * 60) <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.append(mp)
to_fetch.extend(all)
fetched = self._Fetch(to_fetch)
_PostRepoFetch(rp, opt.no_repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
return
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
mp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1)
_ReloadManifest(self)
mp = self.manifest.manifestProject
all = self.GetProjects(args, missing_ok=True)
missing = []
for project in all:
if project.gitdir not in fetched:
missing.append(project)
self._Fetch(missing)
if self.manifest.IsMirror:
# bail out now, we have no working tree
return
if self.UpdateProjectList():
sys.exit(1)
syncbuf = SyncBuffer(mp.config,
detach_head = opt.detach_head)
pm = Progress('Syncing work tree', len(all))
for project in all:
pm.update()
if project.worktree:
project.Sync_LocalHalf(syncbuf)
pm.end()
print >>sys.stderr
if not syncbuf.Finish():
sys.exit(1)
