def Execute(self, opt, args): nb = args[0] err = [] projects = [] if not opt.all: projects = args[1:] if len(projects) < 1: projects = ["."] # start it in the local project by default all_projects = self.GetProjects(projects, missing_ok=bool(self.gitc_manifest)) # This must happen after we find all_projects, since GetProjects may need # the local directory, which will disappear once we save the GITC manifest. if self.gitc_manifest: gitc_projects = self.GetProjects( projects, manifest=self.gitc_manifest, missing_ok=True ) for project in gitc_projects: if project.old_revision: project.already_synced = True else: project.already_synced = False project.old_revision = project.revisionExpr project.revisionExpr = None # Save the GITC manifest. gitc_utils.save_manifest(self.gitc_manifest) # Make sure we have a valid CWD if not os.path.exists(os.getcwd()): os.chdir(self.manifest.topdir) pm = Progress("Starting %s" % nb, len(all_projects)) for project in all_projects: pm.update() if self.gitc_manifest: gitc_project = self.gitc_manifest.paths[project.relpath] # Sync projects that have not been opened. if not gitc_project.already_synced: proj_localdir = os.path.join( self.gitc_manifest.gitc_client_dir, project.relpath ) project.worktree = proj_localdir if not os.path.exists(proj_localdir): os.makedirs(proj_localdir) project.Sync_NetworkHalf() sync_buf = SyncBuffer(self.manifest.manifestProject.config) project.Sync_LocalHalf(sync_buf) project.revisionId = gitc_project.old_revision # If the current revision is immutable, such as a SHA1, a tag or # a change, then we can't push back to it. Substitute with # dest_branch, if defined; or with manifest default revision instead. branch_merge = "" if IsImmutable(project.revisionExpr): if project.dest_branch: branch_merge = project.dest_branch else: branch_merge = self.manifest.default.revisionExpr if not project.StartBranch( nb, branch_merge=branch_merge, revision=opt.revision ): err.append(project) pm.end() if err: for p in err: print("error: %s/: cannot start %s" % (p.relpath, nb), file=sys.stderr) sys.exit(1)
def _CheckoutOne(self, opt, project, lock, pm, err_event, err_results): """Checkout work tree for one project Args: opt: Program options returned from optparse. See _Options(). project: Project object for the project to checkout. lock: Lock for accessing objects that are shared amongst multiple _CheckoutWorker() threads. pm: Instance of a Project object. We will call pm.update() (with our lock held). err_event: We'll set this event in the case of an error (after printing out info about the error). err_results: A list of strings, paths to git repos where checkout failed. Returns: Whether the fetch was successful. """ # We'll set to true once we've locked the lock. did_lock = False # Encapsulate everything in a try/except/finally so that: # - We always set err_event in the case of an exception. # - We always make sure we unlock the lock if we locked it. start = time.time() syncbuf = SyncBuffer(self.manifest.manifestProject.config, detach_head=opt.detach_head) success = False try: try: project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync) # Lock around all the rest of the code, since printing, updating a set # and Progress.update() are not thread safe. lock.acquire() success = syncbuf.Finish() did_lock = True if not success: err_event.set() print('error: Cannot checkout %s' % (project.name), file=sys.stderr) raise _CheckoutError() pm.update(msg=project.name) except _CheckoutError: pass except Exception as e: print('error: Cannot checkout %s: %s: %s' % (project.name, type(e).__name__, str(e)), file=sys.stderr) err_event.set() raise finally: if did_lock: if not success: err_results.append(project.relpath) lock.release() finish = time.time() self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, start, finish, success) return success
def Execute(self, opt, args): if opt.jobs: self.jobs = opt.jobs if self.jobs > 1: soft_limit, _ = _rlimit_nofile() self.jobs = min(self.jobs, (soft_limit - 5) / 3) if opt.network_only and opt.detach_head: print('error: cannot combine -n and -d', file=sys.stderr) sys.exit(1) if opt.network_only and opt.local_only: print('error: cannot combine -n and -l', file=sys.stderr) sys.exit(1) if opt.manifest_name and opt.smart_sync: print('error: cannot combine -m and -s', file=sys.stderr) sys.exit(1) if opt.manifest_name and opt.smart_tag: print('error: cannot combine -m and -t', file=sys.stderr) sys.exit(1) if opt.manifest_server_username or opt.manifest_server_password: if not (opt.smart_sync or opt.smart_tag): print('error: -u and -p may only be combined with -s or -t', file=sys.stderr) sys.exit(1) if None in [ opt.manifest_server_username, opt.manifest_server_password ]: print('error: both -u and -p must be given', file=sys.stderr) sys.exit(1) if opt.manifest_name: self.manifest.Override(opt.manifest_name) manifest_name = opt.manifest_name smart_sync_manifest_name = "smart_sync_override.xml" smart_sync_manifest_path = os.path.join( self.manifest.manifestProject.worktree, smart_sync_manifest_name) if opt.smart_sync or opt.smart_tag: if not self.manifest.manifest_server: print( 'error: cannot smart sync: no manifest server defined in ' 'manifest', file=sys.stderr) sys.exit(1) manifest_server = self.manifest.manifest_server if not opt.quiet: print('Using manifest server %s' % manifest_server) if not '@' in manifest_server: username = None password = None if opt.manifest_server_username and opt.manifest_server_password: username = opt.manifest_server_username password = opt.manifest_server_password else: try: info = netrc.netrc() except IOError: # .netrc file does not exist or could not be opened pass else: try: parse_result = urllib.parse.urlparse( manifest_server) if parse_result.hostname: auth = info.authenticators( parse_result.hostname) if auth: username, _account, password = auth else: print( 'No credentials found for %s in .netrc' % parse_result.hostname, file=sys.stderr) except netrc.NetrcParseError as e: print('Error parsing .netrc file: %s' % e, file=sys.stderr) if (username and password): manifest_server = manifest_server.replace( '://', '://%s:%s@' % (username, password), 1) transport = PersistentTransport(manifest_server) if manifest_server.startswith('persistent-'): manifest_server = manifest_server[len('persistent-'):] try: server = xmlrpc.client.Server(manifest_server, transport=transport) if opt.smart_sync: p = self.manifest.manifestProject b = p.GetBranch(p.CurrentBranch) branch = b.merge if branch.startswith(R_HEADS): branch = branch[len(R_HEADS):] env = os.environ.copy() if 'SYNC_TARGET' in env: target = env['SYNC_TARGET'] [success, manifest_str ] = server.GetApprovedManifest(branch, target) elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env: target = '%s-%s' % (env['TARGET_PRODUCT'], env['TARGET_BUILD_VARIANT']) [success, manifest_str ] = server.GetApprovedManifest(branch, target) else: [success, manifest_str] = server.GetApprovedManifest(branch) else: assert (opt.smart_tag) [success, manifest_str] = server.GetManifest(opt.smart_tag) if success: manifest_name = smart_sync_manifest_name try: f = open(smart_sync_manifest_path, 'w') try: f.write(manifest_str) finally: f.close() except IOError as e: print('error: cannot write manifest to %s:\n%s' % (smart_sync_manifest_path, e), file=sys.stderr) sys.exit(1) self._ReloadManifest(manifest_name) else: print('error: manifest server RPC call failed: %s' % manifest_str, file=sys.stderr) sys.exit(1) except (socket.error, IOError, xmlrpc.client.Fault) as e: print('error: cannot connect to manifest server %s:\n%s' % (self.manifest.manifest_server, e), file=sys.stderr) sys.exit(1) except xmlrpc.client.ProtocolError as e: print('error: cannot connect to manifest server %s:\n%d %s' % (self.manifest.manifest_server, e.errcode, e.errmsg), file=sys.stderr) sys.exit(1) else: # Not smart sync or smart tag mode if os.path.isfile(smart_sync_manifest_path): try: os.remove(smart_sync_manifest_path) except OSError as e: print( 'error: failed to remove existing smart sync override manifest: %s' % e, file=sys.stderr) rp = self.manifest.repoProject rp.PreSync() mp = self.manifest.manifestProject mp.PreSync() if opt.repo_upgraded: _PostRepoUpgrade(self.manifest, quiet=opt.quiet) if not opt.local_only: mp.Sync_NetworkHalf(quiet=opt.quiet, current_branch_only=opt.current_branch_only, no_tags=opt.no_tags, optimized_fetch=opt.optimized_fetch) if mp.HasChanges: syncbuf = SyncBuffer(mp.config) mp.Sync_LocalHalf(syncbuf) if not syncbuf.Finish(): sys.exit(1) self._ReloadManifest(manifest_name) if opt.jobs is None: self.jobs = self.manifest.default.sync_j if self.gitc_manifest: gitc_manifest_projects = self.GetProjects(args, missing_ok=True) gitc_projects = [] opened_projects = [] for project in gitc_manifest_projects: if project.relpath in self.gitc_manifest.paths and \ self.gitc_manifest.paths[project.relpath].old_revision: opened_projects.append(project.relpath) else: gitc_projects.append(project.relpath) if not args: gitc_projects = None if gitc_projects != [] and not opt.local_only: print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name) manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name) if manifest_name: manifest.Override(manifest_name) else: manifest.Override(self.manifest.manifestFile) gitc_utils.generate_gitc_manifest(self.gitc_manifest, manifest, gitc_projects) print('GITC client successfully synced.') # The opened projects need to be synced as normal, therefore we # generate a new args list to represent the opened projects. # TODO: make this more reliable -- if there's a project name/path overlap, # this may choose the wrong project. args = [ os.path.relpath(self.manifest.paths[p].worktree, os.getcwd()) for p in opened_projects ] if not args: return all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=opt.fetch_submodules) self._fetch_times = _FetchTimes(self.manifest) if not opt.local_only: to_fetch = [] now = time.time() if _ONE_DAY_S <= (now - rp.LastFetch): to_fetch.append(rp) to_fetch.extend(all_projects) to_fetch.sort(key=self._fetch_times.Get, reverse=True) fetched = self._Fetch(to_fetch, opt) _PostRepoFetch(rp, opt.no_repo_verify) if opt.network_only: # bail out now; the rest touches the working tree return # Iteratively fetch missing and/or nested unregistered submodules previously_missing_set = set() while True: self._ReloadManifest(manifest_name) all_projects = self.GetProjects( args, missing_ok=True, submodules_ok=opt.fetch_submodules) missing = [] for project in all_projects: if project.gitdir not in fetched: missing.append(project) if not missing: break # Stop us from non-stopped fetching actually-missing repos: If set of # missing repos has not been changed from last fetch, we break. missing_set = set(p.name for p in missing) if previously_missing_set == missing_set: break previously_missing_set = missing_set fetched.update(self._Fetch(missing, opt)) if self.manifest.IsMirror or self.manifest.IsArchive: # bail out now, we have no working tree return if self.UpdateProjectList(): sys.exit(1) syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head) pm = Progress('Syncing work tree', len(all_projects)) for project in all_projects: pm.update() if project.worktree: project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync) pm.end() print(file=sys.stderr) if not syncbuf.Finish(): sys.exit(1) # If there's a notice that's supposed to print at the end of the sync, print # it now... if self.manifest.notice: print(self.manifest.notice)
def _SyncManifest(self, opt): m = self.manifest.manifestProject is_new = not m.Exists if is_new: if not opt.manifest_url: print('fatal: manifest url (-u) is required.', file=sys.stderr) sys.exit(1) if not opt.quiet: print('Get %s' % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url), file=sys.stderr) m._InitGitDir() if opt.manifest_branch: m.revisionExpr = opt.manifest_branch else: m.revisionExpr = 'refs/heads/master' else: if opt.manifest_branch: m.revisionExpr = opt.manifest_branch else: m.PreSync() if opt.manifest_url: r = m.GetRemote(m.remote.name) r.url = opt.manifest_url r.ResetFetch() r.Save() groups = re.split(r'[,\s]+', opt.groups) all_platforms = ['linux', 'darwin'] platformize = lambda x: 'platform-' + x if opt.platform == 'auto': if (not opt.mirror and not m.config.GetString('repo.mirror') == 'true'): groups.append(platformize(platform.system().lower())) elif opt.platform == 'all': groups.extend(map(platformize, all_platforms)) elif opt.platform in all_platforms: groups.extend(platformize(opt.platform)) elif opt.platform != 'none': print('fatal: invalid platform flag', file=sys.stderr) sys.exit(1) groups = [x for x in groups if x] groupstr = ','.join(groups) if opt.platform == 'auto' and groupstr == 'all,-notdefault,platform-' + platform.system().lower(): groupstr = None m.config.SetString('manifest.groups', groupstr) if opt.reference: m.config.SetString('repo.reference', opt.reference) if opt.mirror: if is_new: m.config.SetString('repo.mirror', 'true') else: print('fatal: --mirror not supported on existing client', file=sys.stderr) sys.exit(1) if not m.Sync_NetworkHalf(is_new=is_new): r = m.GetRemote(m.remote.name) print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr) # Better delete the manifest git dir if we created it; otherwise next # time (when user fixes problems) we won't go through the "is_new" logic. if is_new: shutil.rmtree(m.gitdir) sys.exit(1) if opt.manifest_branch: m.MetaBranchSwitch(opt.manifest_branch) syncbuf = SyncBuffer(m.config) m.Sync_LocalHalf(syncbuf) syncbuf.Finish() if is_new or m.CurrentBranch is None: if not m.StartBranch('default'): print('fatal: cannot create default in manifest', file=sys.stderr) sys.exit(1)
def Execute(self, opt, args): if opt.jobs: self.jobs = opt.jobs if self.jobs > 1: soft_limit, _ = _rlimit_nofile() self.jobs = min(self.jobs, (soft_limit - 5) / 3) if opt.network_only and opt.detach_head: print >> sys.stderr, 'error: cannot combine -n and -d' sys.exit(1) if opt.network_only and opt.local_only: print >> sys.stderr, 'error: cannot combine -n and -l' sys.exit(1) if opt.manifest_name and opt.smart_sync: print >> sys.stderr, 'error: cannot combine -m and -s' sys.exit(1) if opt.manifest_name and opt.smart_tag: print >> sys.stderr, 'error: cannot combine -m and -t' sys.exit(1) if opt.manifest_name: self.manifest.Override(opt.manifest_name) if opt.smart_sync or opt.smart_tag: if not self.manifest.manifest_server: print >>sys.stderr, \ 'error: cannot smart sync: no manifest server defined in manifest' sys.exit(1) try: server = xmlrpclib.Server(self.manifest.manifest_server) if opt.smart_sync: p = self.manifest.manifestProject b = p.GetBranch(p.CurrentBranch) branch = b.merge if branch.startswith(R_HEADS): branch = branch[len(R_HEADS):] env = os.environ.copy() if (env.has_key('TARGET_PRODUCT') and env.has_key('TARGET_BUILD_VARIANT')): target = '%s-%s' % (env['TARGET_PRODUCT'], env['TARGET_BUILD_VARIANT']) [success, manifest_str ] = server.GetApprovedManifest(branch, target) else: [success, manifest_str] = server.GetApprovedManifest(branch) else: assert (opt.smart_tag) [success, manifest_str] = server.GetManifest(opt.smart_tag) if success: manifest_name = "smart_sync_override.xml" manifest_path = os.path.join( self.manifest.manifestProject.worktree, manifest_name) try: f = open(manifest_path, 'w') try: f.write(manifest_str) finally: f.close() except IOError: print >>sys.stderr, 'error: cannot write manifest to %s' % \ manifest_path sys.exit(1) self.manifest.Override(manifest_name) else: print >> sys.stderr, 'error: %s' % manifest_str sys.exit(1) except socket.error: print >> sys.stderr, 'error: cannot connect to manifest server %s' % ( self.manifest.manifest_server) sys.exit(1) rp = self.manifest.repoProject rp.PreSync() mp = self.manifest.manifestProject mp.PreSync() if opt.repo_upgraded: _PostRepoUpgrade(self.manifest) if not opt.local_only: mp.Sync_NetworkHalf(quiet=opt.quiet, current_branch_only=opt.current_branch_only) if mp.HasChanges: syncbuf = SyncBuffer(mp.config) mp.Sync_LocalHalf(syncbuf) if not syncbuf.Finish(): sys.exit(1) self.manifest._Unload() if opt.jobs is None: self.jobs = self.manifest.default.sync_j all = self.GetProjects(args, missing_ok=True) if not opt.local_only: to_fetch = [] now = time.time() if (24 * 60 * 60) <= (now - rp.LastFetch): to_fetch.append(rp) to_fetch.extend(all) fetched = self._Fetch(to_fetch, opt) _PostRepoFetch(rp, opt.no_repo_verify) if opt.network_only: # bail out now; the rest touches the working tree return self.manifest._Unload() all = self.GetProjects(args, missing_ok=True) missing = [] for project in all: if project.gitdir not in fetched: missing.append(project) self._Fetch(missing, opt) if self.manifest.IsMirror: # bail out now, we have no working tree return if self.UpdateProjectList(): sys.exit(1) syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head) pm = Progress('Syncing work tree', len(all)) for project in all: pm.update() if project.worktree: project.Sync_LocalHalf(syncbuf) pm.end() print >> sys.stderr if not syncbuf.Finish(): sys.exit(1) # If there's a notice that's supposed to print at the end of the sync, print # it now... if self.manifest.notice: print self.manifest.notice if not opt.local_only: self.svn()
def _SyncManifest(self, opt): m = self.manifest.manifestProject is_new = not m.Exists if is_new: if not opt.manifest_url: print("fatal: manifest url (-u) is required.", file=sys.stderr) sys.exit(1) if not opt.quiet: print( "Downloading manifest from %s" % (GitConfig.ForUser().UrlInsteadOf(opt.manifest_url),), file=sys.stderr, ) # The manifest project object doesn't keep track of the path on the # server where this git is located, so let's save that here. mirrored_manifest_git = None if opt.reference: manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:] mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path) if not mirrored_manifest_git.endswith(".git"): mirrored_manifest_git += ".git" if not os.path.exists(mirrored_manifest_git): mirrored_manifest_git = os.path.join( opt.reference, ".repo/manifests.git" ) m._InitGitDir(mirror_git=mirrored_manifest_git) self._ConfigureDepth(opt) # Set the remote URL before the remote branch as we might need it below. if opt.manifest_url: r = m.GetRemote(m.remote.name) r.url = opt.manifest_url r.ResetFetch() r.Save() if opt.manifest_branch: m.revisionExpr = opt.manifest_branch else: if is_new: default_branch = m.ResolveRemoteHead() if default_branch is None: # If the remote doesn't have HEAD configured, default to master. default_branch = "refs/heads/master" m.revisionExpr = default_branch else: m.PreSync() groups = re.split(r"[,\s]+", opt.groups) all_platforms = ["linux", "darwin", "windows"] platformize = lambda x: "platform-" + x if opt.platform == "auto": if not opt.mirror and not m.config.GetString("repo.mirror") == "true": groups.append(platformize(platform.system().lower())) elif opt.platform == "all": groups.extend(map(platformize, all_platforms)) elif opt.platform in all_platforms: groups.append(platformize(opt.platform)) elif opt.platform != "none": print("fatal: invalid platform flag", file=sys.stderr) sys.exit(1) groups = [x for x in groups if x] groupstr = ",".join(groups) if ( opt.platform == "auto" and groupstr == "default,platform-" + platform.system().lower() ): groupstr = None m.config.SetString("manifest.groups", groupstr) if opt.reference: m.config.SetString("repo.reference", opt.reference) if opt.dissociate: m.config.SetString("repo.dissociate", "true") if opt.worktree: if opt.mirror: print( "fatal: --mirror and --worktree are incompatible", file=sys.stderr ) sys.exit(1) if opt.submodules: print( "fatal: --submodules and --worktree are incompatible", file=sys.stderr, ) sys.exit(1) m.config.SetString("repo.worktree", "true") if is_new: m.use_git_worktrees = True print("warning: --worktree is experimental!", file=sys.stderr) if opt.archive: if is_new: m.config.SetString("repo.archive", "true") else: print( "fatal: --archive is only supported when initializing a new " "workspace.", file=sys.stderr, ) print( "Either delete the .repo folder in this workspace, or initialize " "in another location.", file=sys.stderr, ) sys.exit(1) if opt.mirror: if is_new: m.config.SetString("repo.mirror", "true") else: print( "fatal: --mirror is only supported when initializing a new " "workspace.", file=sys.stderr, ) print( "Either delete the .repo folder in this workspace, or initialize " "in another location.", file=sys.stderr, ) sys.exit(1) if opt.partial_clone: if opt.mirror: print( "fatal: --mirror and --partial-clone are mutually exclusive", file=sys.stderr, ) sys.exit(1) m.config.SetString("repo.partialclone", "true") if opt.clone_filter: m.config.SetString("repo.clonefilter", opt.clone_filter) else: opt.clone_filter = None if opt.clone_bundle is None: opt.clone_bundle = False if opt.partial_clone else True else: m.config.SetString( "repo.clonebundle", "true" if opt.clone_bundle else "false" ) if opt.submodules: m.config.SetString("repo.submodules", "true") if not m.Sync_NetworkHalf( is_new=is_new, quiet=opt.quiet, verbose=opt.verbose, clone_bundle=opt.clone_bundle, current_branch_only=opt.current_branch_only, tags=opt.tags, submodules=opt.submodules, clone_filter=opt.clone_filter, ): r = m.GetRemote(m.remote.name) print("fatal: cannot obtain manifest %s" % r.url, file=sys.stderr) # Better delete the manifest git dir if we created it; otherwise next # time (when user fixes problems) we won't go through the "is_new" logic. if is_new: platform_utils.rmtree(m.gitdir) sys.exit(1) if opt.manifest_branch: m.MetaBranchSwitch(submodules=opt.submodules) syncbuf = SyncBuffer(m.config) m.Sync_LocalHalf(syncbuf, submodules=opt.submodules) syncbuf.Finish() if is_new or m.CurrentBranch is None: if not m.StartBranch("default"): print("fatal: cannot create default in manifest", file=sys.stderr) sys.exit(1)
def Execute(self, opt, args): if opt.jobs: self.jobs = opt.jobs if self.jobs > 1: soft_limit, _ = _rlimit_nofile() self.jobs = min(self.jobs, (soft_limit - 5) / 3) if opt.network_only and opt.detach_head: print('error: cannot combine -n and -d', file=sys.stderr) sys.exit(1) if opt.network_only and opt.local_only: print('error: cannot combine -n and -l', file=sys.stderr) sys.exit(1) if opt.manifest_name and opt.smart_sync: print('error: cannot combine -m and -s', file=sys.stderr) sys.exit(1) if opt.manifest_name and opt.smart_tag: print('error: cannot combine -m and -t', file=sys.stderr) sys.exit(1) if opt.manifest_server_username or opt.manifest_server_password: if not (opt.smart_sync or opt.smart_tag): print('error: -u and -p may only be combined with -s or -t', file=sys.stderr) sys.exit(1) if None in [ opt.manifest_server_username, opt.manifest_server_password ]: print('error: both -u and -p must be given', file=sys.stderr) sys.exit(1) if opt.manifest_name: self.manifest.Override(opt.manifest_name) if opt.smart_sync or opt.smart_tag: if not self.manifest.manifest_server: print( 'error: cannot smart sync: no manifest server defined in' 'manifest', file=sys.stderr) sys.exit(1) manifest_server = self.manifest.manifest_server if not '@' in manifest_server: username = None password = None if opt.manifest_server_username and opt.manifest_server_password: username = opt.manifest_server_username password = opt.manifest_server_password else: try: info = netrc.netrc() except IOError: print( '.netrc file does not exist or could not be opened', file=sys.stderr) else: try: parse_result = urlparse.urlparse(manifest_server) if parse_result.hostname: username, _account, password = \ info.authenticators(parse_result.hostname) except TypeError: # TypeError is raised when the given hostname is not present # in the .netrc file. print('No credentials found for %s in .netrc' % parse_result.hostname, file=sys.stderr) except netrc.NetrcParseError as e: print('Error parsing .netrc file: %s' % e, file=sys.stderr) if (username and password): manifest_server = manifest_server.replace( '://', '://%s:%s@' % (username, password), 1) try: server = xmlrpclib.Server(manifest_server) if opt.smart_sync: p = self.manifest.manifestProject b = p.GetBranch(p.CurrentBranch) branch = b.merge if branch.startswith(R_HEADS): branch = branch[len(R_HEADS):] env = os.environ.copy() if (env.has_key('TARGET_PRODUCT') and env.has_key('TARGET_BUILD_VARIANT')): target = '%s-%s' % (env['TARGET_PRODUCT'], env['TARGET_BUILD_VARIANT']) [success, manifest_str ] = server.GetApprovedManifest(branch, target) else: [success, manifest_str] = server.GetApprovedManifest(branch) else: assert (opt.smart_tag) [success, manifest_str] = server.GetManifest(opt.smart_tag) if success: manifest_name = "smart_sync_override.xml" manifest_path = os.path.join( self.manifest.manifestProject.worktree, manifest_name) try: f = open(manifest_path, 'w') try: f.write(manifest_str) finally: f.close() except IOError: print('error: cannot write manifest to %s' % manifest_path, file=sys.stderr) sys.exit(1) self.manifest.Override(manifest_name) else: print('error: %s' % manifest_str, file=sys.stderr) sys.exit(1) except (socket.error, IOError, xmlrpclib.Fault) as e: print('error: cannot connect to manifest server %s:\n%s' % (self.manifest.manifest_server, e), file=sys.stderr) sys.exit(1) except xmlrpclib.ProtocolError as e: print('error: cannot connect to manifest server %s:\n%d %s' % (self.manifest.manifest_server, e.errcode, e.errmsg), file=sys.stderr) sys.exit(1) rp = self.manifest.repoProject rp.PreSync() mp = self.manifest.manifestProject mp.PreSync() if opt.repo_upgraded: _PostRepoUpgrade(self.manifest, quiet=opt.quiet) if not opt.local_only: mp.Sync_NetworkHalf(quiet=opt.quiet, current_branch_only=opt.current_branch_only) if mp.HasChanges: syncbuf = SyncBuffer(mp.config) mp.Sync_LocalHalf(syncbuf) if not syncbuf.Finish(): sys.exit(1) self.manifest._Unload() if opt.jobs is None: self.jobs = self.manifest.default.sync_j all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=opt.fetch_submodules) self._fetch_times = _FetchTimes(self.manifest) if not opt.local_only: to_fetch = [] now = time.time() if _ONE_DAY_S <= (now - rp.LastFetch): to_fetch.append(rp) to_fetch.extend(all_projects) to_fetch.sort(key=self._fetch_times.Get, reverse=True) fetched = self._Fetch(to_fetch, opt) _PostRepoFetch(rp, opt.no_repo_verify) if opt.network_only: # bail out now; the rest touches the working tree return # Iteratively fetch missing and/or nested unregistered submodules previously_missing_set = set() while True: self.manifest._Unload() all_projects = self.GetProjects( args, missing_ok=True, submodules_ok=opt.fetch_submodules) missing = [] for project in all_projects: if project.gitdir not in fetched: missing.append(project) if not missing: break # Stop us from non-stopped fetching actually-missing repos: If set of # missing repos has not been changed from last fetch, we break. missing_set = set(p.name for p in missing) if previously_missing_set == missing_set: break previously_missing_set = missing_set fetched.update(self._Fetch(missing, opt)) if self.manifest.IsMirror: # bail out now, we have no working tree return if self.UpdateProjectList(): sys.exit(1) syncbuf = SyncBuffer(mp.config, detach_head=opt.detach_head) pm = Progress('Syncing work tree', len(all_projects)) for project in all_projects: pm.update() if project.worktree: project.Sync_LocalHalf(syncbuf) pm.end() print(file=sys.stderr) if not syncbuf.Finish(): sys.exit(1) # If there's a notice that's supposed to print at the end of the sync, print # it now... if self.manifest.notice: print(self.manifest.notice)
def _SyncManifest(self, opt): m = self.manifest.manifestProject is_new = not m.Exists if is_new: if not opt.manifest_url: print('fatal: manifest url (-u) is required.', file=sys.stderr) sys.exit(1) if not opt.quiet: print('Get %s' % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url), file=sys.stderr) # The manifest project object doesn't keep track of the path on the # server where this git is located, so let's save that here. mirrored_manifest_git = None if opt.reference: manifest_git_path = urllib.parse.urlparse( opt.manifest_url).path[1:] mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path) if not mirrored_manifest_git.endswith(".git"): mirrored_manifest_git += ".git" if not os.path.exists(mirrored_manifest_git): mirrored_manifest_git = os.path.join( opt.reference, '.repo/manifests.git') m._InitGitDir(mirror_git=mirrored_manifest_git) if opt.manifest_branch: m.revisionExpr = opt.manifest_branch else: m.revisionExpr = 'refs/heads/master' else: if opt.manifest_branch: m.revisionExpr = opt.manifest_branch else: m.PreSync() if opt.manifest_url: r = m.GetRemote(m.remote.name) r.url = opt.manifest_url r.ResetFetch() r.Save() groups = re.split(r'[,\s]+', opt.groups) all_platforms = ['linux', 'darwin', 'windows'] platformize = lambda x: 'platform-' + x if opt.platform == 'auto': if (not opt.mirror and not m.config.GetString('repo.mirror') == 'true'): groups.append(platformize(platform.system().lower())) elif opt.platform == 'all': groups.extend(map(platformize, all_platforms)) elif opt.platform in all_platforms: groups.append(platformize(opt.platform)) elif opt.platform != 'none': print('fatal: invalid platform flag', file=sys.stderr) sys.exit(1) groups = [x for x in groups if x] groupstr = ','.join(groups) if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system( ).lower(): groupstr = None m.config.SetString('manifest.groups', groupstr) if opt.reference: m.config.SetString('repo.reference', opt.reference) if opt.archive: if is_new: m.config.SetString('repo.archive', 'true') else: print( 'fatal: --archive is only supported when initializing a new ' 'workspace.', file=sys.stderr) print( 'Either delete the .repo folder in this workspace, or initialize ' 'in another location.', file=sys.stderr) sys.exit(1) if opt.mirror: if is_new: m.config.SetString('repo.mirror', 'true') else: print( 'fatal: --mirror is only supported when initializing a new ' 'workspace.', file=sys.stderr) print( 'Either delete the .repo folder in this workspace, or initialize ' 'in another location.', file=sys.stderr) sys.exit(1) if opt.submodules: m.config.SetString('repo.submodules', 'true') if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, clone_bundle=not opt.no_clone_bundle, current_branch_only=opt.current_branch_only, no_tags=opt.no_tags, submodules=opt.submodules): r = m.GetRemote(m.remote.name) print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr) # Better delete the manifest git dir if we created it; otherwise next # time (when user fixes problems) we won't go through the "is_new" logic. if is_new: platform_utils.rmtree(m.gitdir) sys.exit(1) if opt.manifest_branch: m.MetaBranchSwitch(submodules=opt.submodules) syncbuf = SyncBuffer(m.config) m.Sync_LocalHalf(syncbuf, submodules=opt.submodules) syncbuf.Finish() if is_new or m.CurrentBranch is None: if not m.StartBranch('default'): print('fatal: cannot create default in manifest', file=sys.stderr) sys.exit(1)