def _CheckoutOne(self, opt, project, lock, pm, err_event, err_results): """Checkout work tree for one project Args: opt: Program options returned from optparse. See _Options(). project: Project object for the project to checkout. lock: Lock for accessing objects that are shared amongst multiple _CheckoutWorker() threads. pm: Instance of a Project object. We will call pm.update() (with our lock held). err_event: We'll set this event in the case of an error (after printing out info about the error). err_results: A list of strings, paths to git repos where checkout failed. Returns: Whether the fetch was successful. """ # We'll set to true once we've locked the lock. did_lock = False # Encapsulate everything in a try/except/finally so that: # - We always set err_event in the case of an exception. # - We always make sure we unlock the lock if we locked it. start = time.time() syncbuf = SyncBuffer(self.manifest.manifestProject.config, detach_head=opt.detach_head) success = False try: try: project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync) # Lock around all the rest of the code, since printing, updating a set # and Progress.update() are not thread safe. lock.acquire() success = syncbuf.Finish() did_lock = True if not success: err_event.set() print("error: Cannot checkout %s" % (project.name), file=sys.stderr) raise _CheckoutError() pm.update(msg=project.name) except _CheckoutError: pass except Exception as e: print( "error: Cannot checkout %s: %s: %s" % (project.name, type(e).__name__, str(e)), file=sys.stderr, ) err_event.set() raise finally: if did_lock: if not success: err_results.append(project.relpath) lock.release() finish = time.time() self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, start, finish, success) return success
def _SyncManifest(self, opt): m = self.manifest.manifestProject is_new = not m.Exists if is_new: if not opt.manifest_url: print('fatal: manifest url (-u) is required.', file=sys.stderr) sys.exit(1) if not opt.quiet: print('Downloading manifest from %s' % (GitConfig.ForUser().UrlInsteadOf(opt.manifest_url),), file=sys.stderr) # The manifest project object doesn't keep track of the path on the # server where this git is located, so let's save that here. mirrored_manifest_git = None if opt.reference: manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:] mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path) if not mirrored_manifest_git.endswith(".git"): mirrored_manifest_git += ".git" if not os.path.exists(mirrored_manifest_git): mirrored_manifest_git = os.path.join(opt.reference, '.repo/manifests.git') m._InitGitDir(mirror_git=mirrored_manifest_git) self._ConfigureDepth(opt) # Set the remote URL before the remote branch as we might need it below. if opt.manifest_url: r = m.GetRemote(m.remote.name) r.url = opt.manifest_url r.ResetFetch() r.Save() if opt.manifest_branch: m.revisionExpr = opt.manifest_branch else: if is_new: default_branch = m.ResolveRemoteHead() if default_branch is None: # If the remote doesn't have HEAD configured, default to master. default_branch = 'refs/heads/master' m.revisionExpr = default_branch else: m.PreSync() groups = re.split(r'[,\s]+', opt.groups) all_platforms = ['linux', 'darwin', 'windows'] platformize = lambda x: 'platform-' + x if opt.platform == 'auto': if (not opt.mirror and not m.config.GetString('repo.mirror') == 'true'): groups.append(platformize(platform.system().lower())) elif opt.platform == 'all': groups.extend(map(platformize, all_platforms)) elif opt.platform in all_platforms: groups.append(platformize(opt.platform)) elif opt.platform != 'none': print('fatal: invalid platform flag', file=sys.stderr) sys.exit(1) groups = [x for x in groups if x] groupstr = ','.join(groups) if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system().lower(): groupstr = None m.config.SetString('manifest.groups', groupstr) if opt.reference: m.config.SetString('repo.reference', opt.reference) if opt.dissociate: m.config.SetString('repo.dissociate', 'true') if opt.worktree: if opt.mirror: print('fatal: --mirror and --worktree are incompatible', file=sys.stderr) sys.exit(1) if opt.submodules: print('fatal: --submodules and --worktree are incompatible', file=sys.stderr) sys.exit(1) m.config.SetString('repo.worktree', 'true') if is_new: m.use_git_worktrees = True print('warning: --worktree is experimental!', file=sys.stderr) if opt.archive: if is_new: m.config.SetString('repo.archive', 'true') else: print('fatal: --archive is only supported when initializing a new ' 'workspace.', file=sys.stderr) print('Either delete the .repo folder in this workspace, or initialize ' 'in another location.', file=sys.stderr) sys.exit(1) if opt.mirror: if is_new: m.config.SetString('repo.mirror', 'true') else: print('fatal: --mirror is only supported when initializing a new ' 'workspace.', file=sys.stderr) print('Either delete the .repo folder in this workspace, or initialize ' 'in another location.', file=sys.stderr) sys.exit(1) if opt.partial_clone: if opt.mirror: print('fatal: --mirror and --partial-clone are mutually exclusive', file=sys.stderr) sys.exit(1) m.config.SetString('repo.partialclone', 'true') if opt.clone_filter: m.config.SetString('repo.clonefilter', opt.clone_filter) else: opt.clone_filter = None if opt.clone_bundle is None: opt.clone_bundle = False if opt.partial_clone else True else: m.config.SetString('repo.clonebundle', 'true' if opt.clone_bundle else 'false') if opt.submodules: m.config.SetString('repo.submodules', 'true') if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose, clone_bundle=opt.clone_bundle, current_branch_only=opt.current_branch_only, tags=opt.tags, submodules=opt.submodules, clone_filter=opt.clone_filter): r = m.GetRemote(m.remote.name) print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr) # Better delete the manifest git dir if we created it; otherwise next # time (when user fixes problems) we won't go through the "is_new" logic. if is_new: platform_utils.rmtree(m.gitdir) sys.exit(1) if opt.manifest_branch: m.MetaBranchSwitch(submodules=opt.submodules) syncbuf = SyncBuffer(m.config) m.Sync_LocalHalf(syncbuf, submodules=opt.submodules) syncbuf.Finish() if is_new or m.CurrentBranch is None: if not m.StartBranch('default'): print('fatal: cannot create default in manifest', file=sys.stderr) sys.exit(1)
def _SyncManifest(self, opt): m = self.manifest.manifestProject is_new = not m.Exists if is_new: if not opt.manifest_url: print >> sys.stderr, 'fatal: manifest url (-u) is required.' sys.exit(1) if not opt.quiet: print >> sys.stderr, 'Getting manifest ...' print >> sys.stderr, ' from %s' % opt.manifest_url m._InitGitDir() self._ApplyOptions(opt, is_new) if opt.manifest_url: r = m.GetRemote(m.remote.name) r.url = opt.manifest_url r.ResetFetch() r.Save() if opt.reference: m.config.SetString('repo.reference', opt.reference) if opt.mirror: if is_new: m.config.SetString('repo.mirror', 'true') m.config.ClearCache() else: print >> sys.stderr, 'fatal: --mirror not supported on existing client' sys.exit(1) if not m.Sync_NetworkHalf(): r = m.GetRemote(m.remote.name) print >> sys.stderr, 'fatal: cannot obtain manifest %s' % r.url sys.exit(1) if is_new and SubmoduleManifest.IsBare(m): new = self.GetManifest(reparse=True, type=SubmoduleManifest) if m.gitdir != new.manifestProject.gitdir: os.rename(m.gitdir, new.manifestProject.gitdir) new = self.GetManifest(reparse=True, type=SubmoduleManifest) m = new.manifestProject self._ApplyOptions(opt, is_new) if not is_new: # Force the manifest to load if it exists, the old graph # may be needed inside of _ReloadManifest(). # self.manifest.projects syncbuf = SyncBuffer(m.config) m.Sync_LocalHalf(syncbuf) syncbuf.Finish() if isinstance(self.manifest, XmlManifest): self._LinkManifest(opt.manifest_name) _ReloadManifest(self) self._ApplyOptions(opt, is_new) if not self.manifest.InitBranch(): print >> sys.stderr, 'fatal: cannot create branch in manifest' sys.exit(1)
def _SyncManifest(self, opt): m = self.manifest.manifestProject is_new = not m.Exists if is_new: if not opt.manifest_url: print('fatal: manifest url (-u) is required.', file=sys.stderr) sys.exit(1) if not opt.quiet: print('Get %s' % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url), file=sys.stderr) # The manifest project object doesn't keep track of the path on the # server where this git is located, so let's save that here. mirrored_manifest_git = None if opt.reference: manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:] mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path) if not mirrored_manifest_git.endswith(".git"): mirrored_manifest_git += ".git" if not os.path.exists(mirrored_manifest_git): mirrored_manifest_git = os.path.join(opt.reference + '/.repo/manifests.git') m._InitGitDir(mirror_git=mirrored_manifest_git) if opt.manifest_branch: m.revisionExpr = opt.manifest_branch else: m.revisionExpr = 'refs/heads/master' else: if opt.manifest_branch: m.revisionExpr = opt.manifest_branch else: m.PreSync() if opt.manifest_url: r = m.GetRemote(m.remote.name) r.url = opt.manifest_url r.ResetFetch() r.Save() groups = re.split(r'[,\s]+', opt.groups) all_platforms = ['linux', 'darwin', 'windows'] platformize = lambda x: 'platform-' + x if opt.platform == 'auto': if (not opt.mirror and not m.config.GetString('repo.mirror') == 'true'): groups.append(platformize(platform.system().lower())) elif opt.platform == 'all': groups.extend(map(platformize, all_platforms)) elif opt.platform in all_platforms: groups.append(platformize(opt.platform)) elif opt.platform != 'none': print('fatal: invalid platform flag', file=sys.stderr) sys.exit(1) groups = [x for x in groups if x] groupstr = ','.join(groups) if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system().lower(): groupstr = None m.config.SetString('manifest.groups', groupstr) if opt.reference: m.config.SetString('repo.reference', opt.reference) if opt.archive: if is_new: m.config.SetString('repo.archive', 'true') else: print('fatal: --archive is only supported when initializing a new ' 'workspace.', file=sys.stderr) print('Either delete the .repo folder in this workspace, or initialize ' 'in another location.', file=sys.stderr) sys.exit(1) if opt.mirror: if is_new: m.config.SetString('repo.mirror', 'true') else: print('fatal: --mirror is only supported when initializing a new ' 'workspace.', file=sys.stderr) print('Either delete the .repo folder in this workspace, or initialize ' 'in another location.', file=sys.stderr) sys.exit(1) if not m.Sync_NetworkHalf(is_new=is_new): r = m.GetRemote(m.remote.name) print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr) # Better delete the manifest git dir if we created it; otherwise next # time (when user fixes problems) we won't go through the "is_new" logic. if is_new: shutil.rmtree(m.gitdir) sys.exit(1) if opt.manifest_branch: m.MetaBranchSwitch() syncbuf = SyncBuffer(m.config) m.Sync_LocalHalf(syncbuf) syncbuf.Finish() if is_new or m.CurrentBranch is None: if not m.StartBranch('default'): print('fatal: cannot create default in manifest', file=sys.stderr) sys.exit(1)
def Execute(self, opt, args): if opt.jobs: self.jobs = opt.jobs if self.jobs > 1: soft_limit, _ = _rlimit_nofile() self.jobs = min(self.jobs, (soft_limit - 5) / 3) if opt.network_only and opt.detach_head: print('error: cannot combine -n and -d', file=sys.stderr) sys.exit(1) if opt.network_only and opt.local_only: print('error: cannot combine -n and -l', file=sys.stderr) sys.exit(1) if opt.manifest_name and opt.smart_sync: print('error: cannot combine -m and -s', file=sys.stderr) sys.exit(1) if opt.manifest_name and opt.smart_tag: print('error: cannot combine -m and -t', file=sys.stderr) sys.exit(1) if opt.manifest_server_username or opt.manifest_server_password: if not (opt.smart_sync or opt.smart_tag): print('error: -u and -p may only be combined with -s or -t', file=sys.stderr) sys.exit(1) if None in [ opt.manifest_server_username, opt.manifest_server_password ]: print('error: both -u and -p must be given', file=sys.stderr) sys.exit(1) if opt.manifest_name: self.manifest.Override(opt.manifest_name) manifest_name = opt.manifest_name if opt.smart_sync or opt.smart_tag: if not self.manifest.manifest_server: print( 'error: cannot smart sync: no manifest server defined in ' 'manifest', file=sys.stderr) sys.exit(1) manifest_server = self.manifest.manifest_server if not opt.quiet: print('Using manifest server %s' % manifest_server) if not '@' in manifest_server: username = None password = None if opt.manifest_server_username and opt.manifest_server_password: username = opt.manifest_server_username password = opt.manifest_server_password else: try: info = netrc.netrc() except IOError: print( '.netrc file does not exist or could not be opened', file=sys.stderr) else: try: parse_result = urllib.parse.urlparse( manifest_server) if parse_result.hostname: username, _account, password = \ info.authenticators(parse_result.hostname) except TypeError: # TypeError is raised when the given hostname is not present # in the .netrc file. print('No credentials found for %s in .netrc' % parse_result.hostname, file=sys.stderr) except netrc.NetrcParseError as e: print('Error parsing .netrc file: %s' % e, file=sys.stderr) if (username and password): manifest_server = manifest_server.replace( '://', '://%s:%s@' % (username, password), 1) try: server = xmlrpc.client.Server(manifest_server) if opt.smart_sync: p = self.manifest.manifestProject b = p.GetBranch(p.CurrentBranch) branch = b.merge if branch.startswith(R_HEADS): branch = branch[len(R_HEADS):] env = os.environ.copy() if 'SYNC_TARGET' in env: target = env['SYNC_TARGET'] [success, manifest_str ] = server.GetApprovedManifest(branch, target) elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env: target = '%s-%s' % (env['TARGET_PRODUCT'], env['TARGET_BUILD_VARIANT']) [success, manifest_str ] = server.GetApprovedManifest(branch, target) else: [success, manifest_str] = server.GetApprovedManifest(branch) else: assert (opt.smart_tag) [success, manifest_str] = server.GetManifest(opt.smart_tag) if success: manifest_name = "smart_sync_override.xml" manifest_path = os.path.join( self.manifest.manifestProject.worktree, manifest_name) try: f = open(manifest_path, 'w') try: f.write(manifest_str) finally: f.close() except IOError: print('error: cannot write manifest to %s' % manifest_path, file=sys.stderr) sys.exit(1) self._ReloadManifest(manifest_name) else: print('error: manifest server RPC call failed: %s' % manifest_str, file=sys.stderr) sys.exit(1) except (socket.error, IOError, xmlrpc.client.Fault) as e: print('error: cannot connect to manifest server %s:\n%s' % (self.manifest.manifest_server, e), file=sys.stderr) sys.exit(1) except xmlrpc.client.ProtocolError as e: print('error: cannot connect to manifest server %s:\n%d %s' % (self.manifest.manifest_server, e.errcode, e.errmsg), file=sys.stderr) sys.exit(1) rp = self.manifest.repoProject rp.PreSync() mp = self.manifest.manifestProject mp.PreSync() if opt.repo_upgraded: _PostRepoUpgrade(self.manifest, quiet=opt.quiet) if not opt.local_only: mp.Sync_NetworkHalf(quiet=opt.quiet, current_branch_only=opt.current_branch_only, no_tags=opt.no_tags) if mp.HasChanges: syncbuf = SyncBuffer(mp.config) mp.Sync_LocalHalf(syncbuf) if not syncbuf.Finish(): sys.exit(1) self._ReloadManifest(manifest_name) if opt.jobs is None: self.jobs = self.manifest.default.sync_j all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=opt.fetch_submodules) self._fetch_times = _FetchTimes(self.manifest) if not opt.local_only: to_fetch = [] now = time.time() if _ONE_DAY_S <= (now - rp.LastFetch): to_fetch.append(rp) to_fetch.extend(all_projects) to_fetch.sort(key=self._fetch_times.Get, reverse=True) fetched = self._Fetch(to_fetch, opt) _PostRepoFetch(rp, opt.no_repo_verify) if opt.network_only: # bail out now; the rest touches the working tree return # Iteratively fetch missing and/or nested unregistered submodules previously_missing_set = set() while True: self._ReloadManifest(manifest_name) all_projects = self.GetProjects( args, missing_ok=True, submodules_ok=opt.fetch_submodules) missing = [] for project in all_projects: if project.gitdir not in fetched: missing.append(project) if not missing: break # Stop us from non-stopped fetching actually-missing repos: If set of # missing repos has not been changed from last fetch, we break. missing_set = set(p.name for p in missing) if previously_missing_set == missing_set: break previously_missing_set = missing_set fetched.update(self._Fetch(missing, opt)) if self.manifest.IsMirror or self.manifest.IsArchive: # bail out now, we have no working tree return if self.UpdateProjectList(): sys.exit(1) syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head) pm = Progress('Syncing work tree', len(all_projects)) for project in all_projects: pm.update() if project.worktree: project.Sync_LocalHalf(syncbuf) pm.end() print(file=sys.stderr) if not syncbuf.Finish(): sys.exit(1) # If there's a notice that's supposed to print at the end of the sync, print # it now... if self.manifest.notice: print(self.manifest.notice)
def Execute(self, opt, args): if opt.jobs: self.jobs = opt.jobs if self.jobs > 1: soft_limit, _ = _rlimit_nofile() self.jobs = min(self.jobs, (soft_limit - 5) / 3) if opt.network_only and opt.detach_head: print('error: cannot combine -n and -d', file=sys.stderr) sys.exit(1) if opt.network_only and opt.local_only: print('error: cannot combine -n and -l', file=sys.stderr) sys.exit(1) if opt.manifest_name and opt.smart_sync: print('error: cannot combine -m and -s', file=sys.stderr) sys.exit(1) if opt.manifest_name and opt.smart_tag: print('error: cannot combine -m and -t', file=sys.stderr) sys.exit(1) if opt.manifest_server_username or opt.manifest_server_password: if not (opt.smart_sync or opt.smart_tag): print('error: -u and -p may only be combined with -s or -t', file=sys.stderr) sys.exit(1) if None in [opt.manifest_server_username, opt.manifest_server_password]: print('error: both -u and -p must be given', file=sys.stderr) sys.exit(1) if opt.manifest_name: self.manifest.Override(opt.manifest_name) manifest_name = opt.manifest_name smart_sync_manifest_name = "smart_sync_override.xml" smart_sync_manifest_path = os.path.join( self.manifest.manifestProject.worktree, smart_sync_manifest_name) if opt.smart_sync or opt.smart_tag: if not self.manifest.manifest_server: print('error: cannot smart sync: no manifest server defined in ' 'manifest', file=sys.stderr) sys.exit(1) manifest_server = self.manifest.manifest_server if not opt.quiet: print('Using manifest server %s' % manifest_server) if not '@' in manifest_server: username = None password = None if opt.manifest_server_username and opt.manifest_server_password: username = opt.manifest_server_username password = opt.manifest_server_password else: try: info = netrc.netrc() except IOError: # .netrc file does not exist or could not be opened pass else: try: parse_result = urllib.parse.urlparse(manifest_server) if parse_result.hostname: auth = info.authenticators(parse_result.hostname) if auth: username, _account, password = auth else: print('No credentials found for %s in .netrc' % parse_result.hostname, file=sys.stderr) except netrc.NetrcParseError as e: print('Error parsing .netrc file: %s' % e, file=sys.stderr) if (username and password): manifest_server = manifest_server.replace('://', '://%s:%s@' % (username, password), 1) transport = PersistentTransport(manifest_server) if manifest_server.startswith('persistent-'): manifest_server = manifest_server[len('persistent-'):] try: server = xmlrpc.client.Server(manifest_server, transport=transport) if opt.smart_sync: p = self.manifest.manifestProject b = p.GetBranch(p.CurrentBranch) branch = b.merge if branch.startswith(R_HEADS): branch = branch[len(R_HEADS):] env = os.environ.copy() if 'SYNC_TARGET' in env: target = env['SYNC_TARGET'] [success, manifest_str] = server.GetApprovedManifest(branch, target) elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env: target = '%s-%s' % (env['TARGET_PRODUCT'], env['TARGET_BUILD_VARIANT']) [success, manifest_str] = server.GetApprovedManifest(branch, target) else: [success, manifest_str] = server.GetApprovedManifest(branch) else: assert(opt.smart_tag) [success, manifest_str] = server.GetManifest(opt.smart_tag) if success: manifest_name = smart_sync_manifest_name try: f = open(smart_sync_manifest_path, 'w') try: f.write(manifest_str) finally: f.close() except IOError as e: print('error: cannot write manifest to %s:\n%s' % (smart_sync_manifest_path, e), file=sys.stderr) sys.exit(1) self._ReloadManifest(manifest_name) else: print('error: manifest server RPC call failed: %s' % manifest_str, file=sys.stderr) sys.exit(1) except (socket.error, IOError, xmlrpc.client.Fault) as e: print('error: cannot connect to manifest server %s:\n%s' % (self.manifest.manifest_server, e), file=sys.stderr) sys.exit(1) except xmlrpc.client.ProtocolError as e: print('error: cannot connect to manifest server %s:\n%d %s' % (self.manifest.manifest_server, e.errcode, e.errmsg), file=sys.stderr) sys.exit(1) else: # Not smart sync or smart tag mode if os.path.isfile(smart_sync_manifest_path): try: platform_utils.remove(smart_sync_manifest_path) except OSError as e: print('error: failed to remove existing smart sync override manifest: %s' % e, file=sys.stderr) rp = self.manifest.repoProject rp.PreSync() mp = self.manifest.manifestProject mp.PreSync() if opt.repo_upgraded: _PostRepoUpgrade(self.manifest, quiet=opt.quiet) if not opt.local_only: start = time.time() success = mp.Sync_NetworkHalf(quiet=opt.quiet, current_branch_only=opt.current_branch_only, no_tags=opt.no_tags, optimized_fetch=opt.optimized_fetch, submodules=self.manifest.HasSubmodules) finish = time.time() self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK, start, finish, success) if mp.HasChanges: syncbuf = SyncBuffer(mp.config) start = time.time() mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules) clean = syncbuf.Finish() self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL, start, time.time(), clean) if not clean: sys.exit(1) self._ReloadManifest(manifest_name) if opt.jobs is None: self.jobs = self.manifest.default.sync_j if self.gitc_manifest: gitc_manifest_projects = self.GetProjects(args, missing_ok=True) gitc_projects = [] opened_projects = [] for project in gitc_manifest_projects: if project.relpath in self.gitc_manifest.paths and \ self.gitc_manifest.paths[project.relpath].old_revision: opened_projects.append(project.relpath) else: gitc_projects.append(project.relpath) if not args: gitc_projects = None if gitc_projects != [] and not opt.local_only: print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name) manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name) if manifest_name: manifest.Override(manifest_name) else: manifest.Override(self.manifest.manifestFile) gitc_utils.generate_gitc_manifest(self.gitc_manifest, manifest, gitc_projects) print('GITC client successfully synced.') # The opened projects need to be synced as normal, therefore we # generate a new args list to represent the opened projects. # TODO: make this more reliable -- if there's a project name/path overlap, # this may choose the wrong project. args = [os.path.relpath(self.manifest.paths[path].worktree, os.getcwd()) for path in opened_projects] if not args: return all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=opt.fetch_submodules) self._fetch_times = _FetchTimes(self.manifest) if not opt.local_only: to_fetch = [] now = time.time() if _ONE_DAY_S <= (now - rp.LastFetch): to_fetch.append(rp) to_fetch.extend(all_projects) to_fetch.sort(key=self._fetch_times.Get, reverse=True) fetched = self._Fetch(to_fetch, opt) _PostRepoFetch(rp, opt.no_repo_verify) if opt.network_only: # bail out now; the rest touches the working tree return # Iteratively fetch missing and/or nested unregistered submodules previously_missing_set = set() while True: self._ReloadManifest(manifest_name) all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=opt.fetch_submodules) missing = [] for project in all_projects: if project.gitdir not in fetched: missing.append(project) if not missing: break # Stop us from non-stopped fetching actually-missing repos: If set of # missing repos has not been changed from last fetch, we break. missing_set = set(p.name for p in missing) if previously_missing_set == missing_set: break previously_missing_set = missing_set fetched.update(self._Fetch(missing, opt)) if self.manifest.IsMirror or self.manifest.IsArchive: # bail out now, we have no working tree return if self.UpdateProjectList(): sys.exit(1) syncbuf = SyncBuffer(mp.config, detach_head = opt.detach_head) pm = Progress('Syncing work tree', len(all_projects)) for project in all_projects: pm.update() if project.worktree: start = time.time() project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync) self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, start, time.time(), syncbuf.Recently()) pm.end() print(file=sys.stderr) if not syncbuf.Finish(): sys.exit(1) # If there's a notice that's supposed to print at the end of the sync, print # it now... if self.manifest.notice: print(self.manifest.notice)
def Execute(self, opt, args): if opt.network_only and opt.detach_head: print >> sys.stderr, 'error: cannot combine -n and -d' sys.exit(1) if opt.network_only and opt.local_only: print >> sys.stderr, 'error: cannot combine -n and -l' sys.exit(1) rp = self.manifest.repoProject rp.PreSync() mp = self.manifest.manifestProject mp.PreSync() if opt.repo_upgraded: _PostRepoUpgrade(self.manifest) all = self.GetProjects(args, missing_ok=True) if not opt.local_only: to_fetch = [] now = time.time() if (24 * 60 * 60) <= (now - rp.LastFetch): to_fetch.append(rp) to_fetch.append(mp) to_fetch.extend(all) fetched = self._Fetch(to_fetch) _PostRepoFetch(rp, opt.no_repo_verify) if opt.network_only: # bail out now; the rest touches the working tree return if mp.HasChanges: syncbuf = SyncBuffer(mp.config) mp.Sync_LocalHalf(syncbuf) if not syncbuf.Finish(): sys.exit(1) self.manifest._Unload() all = self.GetProjects(args, missing_ok=True) missing = [] for project in all: if project.gitdir not in fetched: missing.append(project) self._Fetch(missing) if self.manifest.IsMirror: # bail out now, we have no working tree return if self.UpdateProjectList(): sys.exit(1) syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head) pm = Progress('Syncing work tree', len(all)) for project in all: pm.update() if project.worktree: project.Sync_LocalHalf(syncbuf) pm.end() print >> sys.stderr if not syncbuf.Finish(): sys.exit(1)
def Execute(self, opt, args): if opt.jobs: self.jobs = 3 if opt.network_only and opt.detach_head: print >> sys.stderr, 'error: cannot combine -n and -d' sys.exit(1) if opt.network_only and opt.local_only: print >> sys.stderr, 'error: cannot combine -n and -l' sys.exit(1) if opt.smart_sync: if not self.manifest.manifest_server: print >>sys.stderr, \ 'error: cannot smart sync: no manifest server defined in manifest' sys.exit(1) try: server = xmlrpclib.Server(self.manifest.manifest_server) p = self.manifest.manifestProject b = p.GetBranch(p.CurrentBranch) branch = b.merge if branch.startswith(R_HEADS): branch = branch[len(R_HEADS):] env = os.environ.copy() if (env.has_key('TARGET_PRODUCT') and env.has_key('TARGET_BUILD_VARIANT')): target = '%s-%s' % (env['TARGET_PRODUCT'], env['TARGET_BUILD_VARIANT']) [success, manifest_str ] = server.GetApprovedManifest(branch, target) else: [success, manifest_str] = server.GetApprovedManifest(branch) if success: manifest_name = "smart_sync_override.xml" manifest_path = os.path.join( self.manifest.manifestProject.worktree, manifest_name) try: f = open(manifest_path, 'w') try: f.write(manifest_str) finally: f.close() except IOError: print >>sys.stderr, 'error: cannot write manifest to %s' % \ manifest_path sys.exit(1) self.manifest.Override(manifest_name) else: print >> sys.stderr, 'error: %s' % manifest_str sys.exit(1) except socket.error: print >> sys.stderr, 'error: cannot connect to manifest server %s' % ( self.manifest.manifest_server) sys.exit(1) rp = self.manifest.repoProject rp.PreSync() mp = self.manifest.manifestProject mp.PreSync() if opt.repo_upgraded: _PostRepoUpgrade(self.manifest) if not opt.local_only: mp.Sync_NetworkHalf(quiet=opt.quiet) if mp.HasChanges: syncbuf = SyncBuffer(mp.config) mp.Sync_LocalHalf(syncbuf) if not syncbuf.Finish(): sys.exit(1) self.manifest._Unload() all = self.GetProjects(args, missing_ok=True) if not opt.local_only: to_fetch = [] now = time.time() if (24 * 60 * 60) <= (now - rp.LastFetch): to_fetch.append(rp) to_fetch.extend(all) fetched = self._Fetch(to_fetch, opt) _PostRepoFetch(rp, True) if opt.network_only: # bail out now; the rest touches the working tree return self.manifest._Unload() all = self.GetProjects(args, missing_ok=True) missing = [] for project in all: if project.gitdir not in fetched: missing.append(project) self._Fetch(missing, opt) if self.manifest.IsMirror: # bail out now, we have no working tree return if self.UpdateProjectList(): sys.exit(1) syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head) pm = Progress('Syncing work tree', len(all)) for project in all: pm.update() if project.worktree: project.Sync_LocalHalf(syncbuf) pm.end() print >> sys.stderr if not syncbuf.Finish(): sys.exit(1) # If there's a notice that's supposed to print at the end of the sync, print # it now... if self.manifest.notice: print self.manifest.notice
def _SyncManifest(self, opt): m = self.manifest.manifestProject is_new = not m.Exists # If repo has already been initialized, we take -u with the absence of # --standalone-manifest to mean "transition to a standard repo set up", # which necessitates starting fresh. # If --standalone-manifest is set, we always tear everything down and start # anew. if not is_new: was_standalone_manifest = m.config.GetString('manifest.standalone') if was_standalone_manifest and not opt.manifest_url: print('fatal: repo was initialized with a standlone manifest, ' 'cannot be re-initialized without --manifest-url/-u') sys.exit(1) if opt.standalone_manifest or (was_standalone_manifest and opt.manifest_url): m.config.ClearCache() if m.gitdir and os.path.exists(m.gitdir): platform_utils.rmtree(m.gitdir) if m.worktree and os.path.exists(m.worktree): platform_utils.rmtree(m.worktree) is_new = not m.Exists if is_new: if not opt.manifest_url: print('fatal: manifest url is required.', file=sys.stderr) sys.exit(1) if not opt.quiet: print('Downloading manifest from %s' % (GitConfig.ForUser().UrlInsteadOf(opt.manifest_url), ), file=sys.stderr) # The manifest project object doesn't keep track of the path on the # server where this git is located, so let's save that here. mirrored_manifest_git = None if opt.reference: manifest_git_path = urllib.parse.urlparse( opt.manifest_url).path[1:] mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path) if not mirrored_manifest_git.endswith(".git"): mirrored_manifest_git += ".git" if not os.path.exists(mirrored_manifest_git): mirrored_manifest_git = os.path.join( opt.reference, '.repo/manifests.git') m._InitGitDir(mirror_git=mirrored_manifest_git) # If standalone_manifest is set, mark the project as "standalone" -- we'll # still do much of the manifests.git set up, but will avoid actual syncs to # a remote. standalone_manifest = False if opt.standalone_manifest: standalone_manifest = True m.config.SetString('manifest.standalone', opt.manifest_url) elif not opt.manifest_url and not opt.manifest_branch: # If -u is set and --standalone-manifest is not, then we're not in # standalone mode. Otherwise, use config to infer what we were in the last # init. standalone_manifest = bool( m.config.GetString('manifest.standalone')) if not standalone_manifest: m.config.SetString('manifest.standalone', None) self._ConfigureDepth(opt) # Set the remote URL before the remote branch as we might need it below. if opt.manifest_url: r = m.GetRemote(m.remote.name) r.url = opt.manifest_url r.ResetFetch() r.Save() if not standalone_manifest: if opt.manifest_branch: if opt.manifest_branch == 'HEAD': opt.manifest_branch = m.ResolveRemoteHead() if opt.manifest_branch is None: print('fatal: unable to resolve HEAD', file=sys.stderr) sys.exit(1) m.revisionExpr = opt.manifest_branch else: if is_new: default_branch = m.ResolveRemoteHead() if default_branch is None: # If the remote doesn't have HEAD configured, default to master. default_branch = 'refs/heads/master' m.revisionExpr = default_branch else: m.PreSync() groups = re.split(r'[,\s]+', opt.groups) all_platforms = ['linux', 'darwin', 'windows'] platformize = lambda x: 'platform-' + x if opt.platform == 'auto': if (not opt.mirror and not m.config.GetString('repo.mirror') == 'true'): groups.append(platformize(platform.system().lower())) elif opt.platform == 'all': groups.extend(map(platformize, all_platforms)) elif opt.platform in all_platforms: groups.append(platformize(opt.platform)) elif opt.platform != 'none': print('fatal: invalid platform flag', file=sys.stderr) sys.exit(1) groups = [x for x in groups if x] groupstr = ','.join(groups) if opt.platform == 'auto' and groupstr == self.manifest.GetDefaultGroupsStr( ): groupstr = None m.config.SetString('manifest.groups', groupstr) if opt.reference: m.config.SetString('repo.reference', opt.reference) if opt.dissociate: m.config.SetBoolean('repo.dissociate', opt.dissociate) if opt.worktree: if opt.mirror: print('fatal: --mirror and --worktree are incompatible', file=sys.stderr) sys.exit(1) if opt.submodules: print('fatal: --submodules and --worktree are incompatible', file=sys.stderr) sys.exit(1) m.config.SetBoolean('repo.worktree', opt.worktree) if is_new: m.use_git_worktrees = True print('warning: --worktree is experimental!', file=sys.stderr) if opt.archive: if is_new: m.config.SetBoolean('repo.archive', opt.archive) else: print( 'fatal: --archive is only supported when initializing a new ' 'workspace.', file=sys.stderr) print( 'Either delete the .repo folder in this workspace, or initialize ' 'in another location.', file=sys.stderr) sys.exit(1) if opt.mirror: if is_new: m.config.SetBoolean('repo.mirror', opt.mirror) else: print( 'fatal: --mirror is only supported when initializing a new ' 'workspace.', file=sys.stderr) print( 'Either delete the .repo folder in this workspace, or initialize ' 'in another location.', file=sys.stderr) sys.exit(1) if opt.partial_clone is not None: if opt.mirror: print( 'fatal: --mirror and --partial-clone are mutually exclusive', file=sys.stderr) sys.exit(1) m.config.SetBoolean('repo.partialclone', opt.partial_clone) if opt.clone_filter: m.config.SetString('repo.clonefilter', opt.clone_filter) elif m.config.GetBoolean('repo.partialclone'): opt.clone_filter = m.config.GetString('repo.clonefilter') else: opt.clone_filter = None if opt.partial_clone_exclude is not None: m.config.SetString('repo.partialcloneexclude', opt.partial_clone_exclude) if opt.clone_bundle is None: opt.clone_bundle = False if opt.partial_clone else True else: m.config.SetBoolean('repo.clonebundle', opt.clone_bundle) if opt.submodules: m.config.SetBoolean('repo.submodules', opt.submodules) if opt.use_superproject is not None: m.config.SetBoolean('repo.superproject', opt.use_superproject) if standalone_manifest: if is_new: manifest_name = 'default.xml' manifest_data = fetch.fetch_file(opt.manifest_url, verbose=opt.verbose) dest = os.path.join(m.worktree, manifest_name) os.makedirs(os.path.dirname(dest), exist_ok=True) with open(dest, 'wb') as f: f.write(manifest_data) return if not m.Sync_NetworkHalf( is_new=is_new, quiet=opt.quiet, verbose=opt.verbose, clone_bundle=opt.clone_bundle, current_branch_only=opt.current_branch_only, tags=opt.tags, submodules=opt.submodules, clone_filter=opt.clone_filter, partial_clone_exclude=self.manifest.PartialCloneExclude): r = m.GetRemote(m.remote.name) print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr) # Better delete the manifest git dir if we created it; otherwise next # time (when user fixes problems) we won't go through the "is_new" logic. if is_new: platform_utils.rmtree(m.gitdir) sys.exit(1) if opt.manifest_branch: m.MetaBranchSwitch(submodules=opt.submodules) syncbuf = SyncBuffer(m.config) m.Sync_LocalHalf(syncbuf, submodules=opt.submodules) syncbuf.Finish() if is_new or m.CurrentBranch is None: if not m.StartBranch('default'): print('fatal: cannot create default in manifest', file=sys.stderr) sys.exit(1)
def _SyncManifest(self, opt): m = self.manifest.manifestProject is_new = not m.Exists if is_new: if not opt.manifest_url: print >> sys.stderr, 'fatal: manifest url (-u) is required.' sys.exit(1) if not opt.quiet: print >>sys.stderr, 'Get %s' \ % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url) m._InitGitDir() if opt.manifest_branch: m.revisionExpr = opt.manifest_branch else: m.revisionExpr = 'refs/heads/master' else: if opt.manifest_branch: m.revisionExpr = opt.manifest_branch else: m.PreSync() if opt.manifest_url: r = m.GetRemote(m.remote.name) r.url = opt.manifest_url r.ResetFetch() r.Save() groups = re.split('[,\s]+', opt.groups) all_platforms = ['linux', 'darwin'] platformize = lambda x: 'platform-' + x if opt.platform == 'auto': if (not opt.mirror and not m.config.GetString('repo.mirror') == 'true'): groups.append(platformize(platform.system().lower())) elif opt.platform == 'all': groups.extend(map(platformize, all_platforms)) elif opt.platform in all_platforms: groups.extend(platformize(opt.platform)) elif opt.platform != 'none': print >> sys.stderr, 'fatal: invalid platform flag' sys.exit(1) groups = [x for x in groups if x] groupstr = ','.join(groups) if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system( ).lower(): groupstr = None m.config.SetString('manifest.groups', groupstr) if opt.reference: m.config.SetString('repo.reference', opt.reference) if opt.mirror: if is_new: m.config.SetString('repo.mirror', 'true') else: print >> sys.stderr, 'fatal: --mirror not supported on existing client' sys.exit(1) if not m.Sync_NetworkHalf(is_new=is_new): r = m.GetRemote(m.remote.name) print >> sys.stderr, 'fatal: cannot obtain manifest %s' % r.url # Better delete the manifest git dir if we created it; otherwise next # time (when user fixes problems) we won't go through the "is_new" logic. if is_new: shutil.rmtree(m.gitdir) sys.exit(1) if opt.manifest_branch: m.MetaBranchSwitch(opt.manifest_branch) syncbuf = SyncBuffer(m.config) m.Sync_LocalHalf(syncbuf) syncbuf.Finish() if is_new or m.CurrentBranch is None: if not m.StartBranch('default'): print >> sys.stderr, 'fatal: cannot create default in manifest' sys.exit(1)