def resolve_refs(self): manifest = self.query_manifest() worker_pool = ThreadPool(8) results = [] # Resolve refnames for p in manifest.getElementsByTagName('project'): name = p.getAttribute('name') remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) # commit ids are already done if repo_manifest.is_commitid(revision): self.debug("%s is already locked to %s; skipping" % (name, revision)) continue # If there's no '/' in the revision, assume it's a head if '/' not in revision: revision = 'refs/heads/%s' % revision self.debug("Getting revision for %s (currently %s)" % (name, revision)) async_result = worker_pool.apply_async(self.resolve_git_ref, (remote_url, revision)) results.append((p, async_result)) for p, result in results: revision = result.get() p.setAttribute('revision', revision)
def resolve_refs(self, manifest): worker_pool = ThreadPool(20) lookup_threads_by_project = {} lookup_threads_by_parameters = {} # Resolve refnames for p in manifest.getElementsByTagName('project'): name = p.getAttribute('name') remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) # commit ids are already done if repo_manifest.is_commitid(revision): self.debug("%s is already locked to %s; skipping" % (name, revision)) continue # gaia is special - make sure we're using the same revision we used # for gaia.json if self.gaia_hg_revision and p.getAttribute('path') == 'gaia' and revision == self.config['gaia_git_branch']: git_rev = self.query_gaia_git_rev() self.info("Using %s for gaia to match %s in gaia.json" % (git_rev, self.gaia_hg_revision)) p.setAttribute('revision', git_rev) continue # If there's no '/' in the revision, assume it's a head if '/' not in revision: revision = 'refs/heads/%s' % revision cache_key = "%s:%s" % (remote_url, revision) # Check to see if we've looked up this revision on this remote # before. If we have, reuse the previous value rather than looking # it up again. This will make sure revisions for the same ref name # are consistent between devices, as long as they use the same # remote/refname. if cache_key in self._git_ref_cache: abs_revision = self._git_ref_cache[cache_key] self.debug( "Reusing previous lookup %s -> %s" % (cache_key, abs_revision)) p.setAttribute('revision', abs_revision) continue # Maybe a thread already exists for this lookup, even if the result has not # yet been retrieved and placed in _git_ref_cache... # Please note result.get() can be called multiple times without problems; # the git command will only be executed once. Therefore we can associate many # projects to the same thread result, without problems later when we call # get() multiple times against the same thread result. if cache_key in lookup_threads_by_parameters: self.debug("Reusing currently running thread to look up %s" % cache_key) lookup_threads_by_project[p] = lookup_threads_by_parameters.get(cache_key) else: async_result = worker_pool.apply_async(self.resolve_git_ref, (remote_url, revision)) lookup_threads_by_parameters[cache_key] = async_result lookup_threads_by_project[p] = async_result # TODO: alert/notify on missing repositories abort = False failed = [] for p, result in lookup_threads_by_project.iteritems(): abs_revision = result.get(timeout=300) remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) if not abs_revision: abort = True self.error("Couldn't resolve reference %s %s" % (remote_url, revision)) failed.append(p) p.setAttribute('revision', abs_revision) if abort: # Write message about how to set up syncing default = repo_manifest.get_default(manifest) for p in failed: if p.hasAttribute('remote'): remote = repo_manifest.get_remote(manifest, p.getAttribute('remote')) else: remote = repo_manifest.get_remote(manifest, default.getAttribute('remote')) new_fetch_url = remote.getAttribute('fetch') orig_fetch_url = self._remote_mappings[new_fetch_url] name = p.getAttribute('name') self.info("needs sync? %s/%s -> %s/%s" % (orig_fetch_url, name, new_fetch_url, name)) self.fatal("couldn't resolve some refs; exiting")
def munge_manifests(self): """ Switch the branched repos to the new branch; lock down third party revisions. """ branch_repos = self.query_branch_repos() dirs = self.query_abs_dirs() new_branch = self.config['branch_name'] unused_manifests = [] if not self.check_existing_branch(new_branch, cwd=dirs['abs_manifest_dir'])[0]: self.fatal("b2g-manifest isn't branched properly! Run --clean-repos --branch-repos") for manifest in self.query_manifests(): self.info("Munging %s..." % manifest) doc = repo_manifest.load_manifest(manifest) try: repo_manifest.get_default(doc).getAttribute("revision") except IndexError: self.info("No default revision; skipping.") unused_manifests.append(manifest) continue for p in doc.getElementsByTagName('project'): name = self._query_repo_name(p.getAttribute('name')) fetch = repo_manifest.get_project_remote_url(doc, p) self.debug("Remote %s Name %s" % (fetch, name)) current_revision = repo_manifest.get_project_revision(doc, p) if repo_manifest.is_commitid(current_revision): self.info("%s: %s is already locked to %s; skipping." % (manifest, name, current_revision)) # I could setAttribute() here, but I very much doubt the # default_revision is a commitid. continue # We've branched this repo; do we set the revision to # new_branch or not? ('fetch' needs to match, since we have # same-named repos with different urls =P ) if name in branch_repos and branch_repos[name]['fetch'] == fetch: orig_branch = branch_repos[name]['branch_revisions'].keys()[0] if manifest in branch_repos[name]['branch_revisions'][orig_branch]: if current_revision != orig_branch: self.fatal("I don't know how we got here, but %s in %s's revision %s is not the branching point %s." % (name, manifest, current_revision, orig_branch)) self.info("Setting %s (%s) to %s (was %s)" % (name, manifest, new_branch, current_revision)) p.setAttribute('revision', new_branch) continue # Should we keep the old branch or lock revision? Doing # the former for now. self.info("%s %s is off a different branch point (%s, not %s). Keeping the old branch..." % (manifest, name, current_revision, orig_branch)) continue if name in self.config['extra_branch_manifest_repos']: p.setAttribute('revision', new_branch) continue # Lock revision? if not self.config["lock_manifest_revisions"]: self.info("%s: Not locking revision for %s due to config." % (manifest, name)) continue lock_revision = self._query_remote_branch_revision(fetch, current_revision, manifest) if lock_revision is not None: p.setAttribute('revision', lock_revision) with self.opened(manifest, open_mode='w') as (fh, err): if err: self.fatal("Can't open %s for writing!" % manifest) else: doc.writexml(fh) fh.close() if self.config["delete_unused_manifests"]: self._delete_unused_manifests(unused_manifests) self.info("TODO: diff, commit, --push!")
def query_branch_repos(self): """ Parse all manifests and build a dictionary of repos with expected revisions and/or branches. The format will be { name: { 'revision': branch, 'fetch': git_url, 'branch_revisions': { revision: [manifest list] # This should only have one key/value pair }, 'all_revisions': { revision: [manifest list] # This will have all key/value pairs }, }, } This depends on the pull action having run at least once. """ if self.branch_repo_dict is not None: return self.branch_repo_dict self.info("Building branch_repo_dict...") branch_repo_dict = { 'b2g-manifest': { 'fetch': self.config['manifest_repo_url'], 'revision': self.config['manifest_repo_revision'], }, } for manifest in self.query_manifests(): self.info("Processing %s" % manifest) doc = repo_manifest.load_manifest(manifest) try: default_revision = repo_manifest.get_default(doc).getAttribute("revision") except IndexError: self.info("Skipping %s (no defaults)" % manifest) continue for p in doc.getElementsByTagName('project'): name = self._query_repo_name(p.getAttribute('name')) fetch = repo_manifest.get_project_remote_url(doc, p) self.debug("Remote %s Name %s" % (fetch, name)) # We branch github.com/mozilla repos only if not self._query_do_branch(fetch, name): continue # Now start building the branch info branch_repo_dict.setdefault(name, {}).setdefault("fetch", fetch) revision = p.getAttribute("revision") if not revision: # fall back to default revision if default_revision: self.info("%s: %s, using default revision (%s)" % (manifest, name, default_revision)) self._add_branch_revision(branch_repo_dict[name], name, default_revision, manifest) else: self.warning("Can't determine revision for %s in %s" % (name, manifest)) elif revision and branch_repo_dict.get(name, {}).get("revision", revision) != revision: self._add_branch_revision(branch_repo_dict[name], name, revision, manifest) else: self._add_branch_revision(branch_repo_dict[name], name, revision, manifest) self.info("Outputting branch_repo_dict:") self.info(pprint.pformat(branch_repo_dict)) message = "" for name, r in branch_repo_dict.iteritems(): if r.get("revision") is None: self.warning("Sanity: No revision set for %s %s; we'll fall back to master" % (name, r["fetch"])) if len(r.get("branch_revisions", {})) > 1: message += "Sanity: Not clear where to branch for %s %s %s\n" % (name, r["fetch"], pprint.pformat(r["branch_revisions"])) if message: self.fatal(message + "Use --branch-order or self.config['no_branch_repos'] to fix!") self.branch_repo_dict = branch_repo_dict return branch_repo_dict
def resolve_refs(self, manifest): worker_pool = ThreadPool(20) results = [] # Resolve refnames for p in manifest.getElementsByTagName('project'): name = p.getAttribute('name') remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) # commit ids are already done if repo_manifest.is_commitid(revision): self.debug("%s is already locked to %s; skipping" % (name, revision)) continue # gaia is special - make sure we're using the same revision we used # for gaia.json if self.gaia_hg_revision and p.getAttribute('path') == 'gaia' and revision == self.config['gaia_git_branch']: if not self.gaia_git_rev: self.gaia_git_rev = self.query_mapper_git_revision( self.config['mapper_url'], self.config['gaia_mapper_project'], self.gaia_hg_revision, ) self.info("Using %s for gaia to match %s in gaia.json" % (self.gaia_git_rev, self.gaia_hg_revision)) p.setAttribute('revision', self.gaia_git_rev) continue # Check to see if we've looked up this revision on this remote # before If we have, reuse the previous value rather than looking # it up again This will make sure revisions for the same ref name # are consistent between devices, as long as they use the same # remote/refname if (remote_url, revision) in self._git_ref_cache: abs_revision = self._git_ref_cache[remote_url, revision] self.info( "Re-using previous lookup %s:%s -> %s" % (remote_url, revision, abs_revision)) p.setAttribute('revision', abs_revision) continue # If there's no '/' in the revision, assume it's a head if '/' not in revision: revision = 'refs/heads/%s' % revision self.debug("Getting revision for %s (currently %s)" % (name, revision)) async_result = worker_pool.apply_async(self.resolve_git_ref, (remote_url, revision)) results.append((p, async_result)) # TODO: alert/notify on missing repositories # TODO: Add external caching abort = False failed = [] for p, result in results: abs_revision = result.get() remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) if not abs_revision: abort = True self.error("Couldn't resolve %s %s" % (remote_url, revision)) failed.append(p) # Save to our cache self._git_ref_cache[remote_url, revision] = abs_revision p.setAttribute('revision', abs_revision) if abort: # Write message about how to set up syncing default = repo_manifest.get_default(manifest) for p in failed: if p.hasAttribute('remote'): remote = repo_manifest.get_remote(manifest, p.getAttribute('remote')) else: remote = repo_manifest.get_remote(manifest, default.getAttribute('remote')) new_fetch_url = remote.getAttribute('fetch') orig_fetch_url = self._remote_mappings[new_fetch_url] name = p.getAttribute('name') self.info("needs sync? %s/%s -> %s/%s" % (orig_fetch_url, name, new_fetch_url, name)) self.fatal("couldn't resolve some refs; exiting")
def resolve_refs(self, manifest): worker_pool = ThreadPool(20) results = [] # Resolve refnames for p in manifest.getElementsByTagName("project"): name = p.getAttribute("name") remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) # commit ids are already done if repo_manifest.is_commitid(revision): self.debug("%s is already locked to %s; skipping" % (name, revision)) continue # gaia is special - make sure we're using the same revision we used # for gaia.json if ( self.gaia_hg_revision and p.getAttribute("path") == "gaia" and revision == self.config["gaia_git_branch"] ): if not self.gaia_git_rev: self.gaia_git_rev = self.query_mapper_git_revision( self.config["mapper_url"], self.config["gaia_mapper_project"], self.gaia_hg_revision ) self.info("Using %s for gaia to match %s in gaia.json" % (self.gaia_git_rev, self.gaia_hg_revision)) p.setAttribute("revision", self.gaia_git_rev) continue # Check to see if we've looked up this revision on this remote # before If we have, reuse the previous value rather than looking # it up again This will make sure revisions for the same ref name # are consistent between devices, as long as they use the same # remote/refname if (remote_url, revision) in self._git_ref_cache: abs_revision = self._git_ref_cache[remote_url, revision] self.info("Re-using previous lookup %s:%s -> %s" % (remote_url, revision, abs_revision)) p.setAttribute("revision", abs_revision) continue # If there's no '/' in the revision, assume it's a head if "/" not in revision: revision = "refs/heads/%s" % revision self.debug("Getting revision for %s (currently %s)" % (name, revision)) async_result = worker_pool.apply_async(self.resolve_git_ref, (remote_url, revision)) results.append((p, async_result)) # TODO: alert/notify on missing repositories # TODO: Add external caching abort = False for p, result in results: abs_revision = result.get() remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) if not abs_revision: abort = True self.error("Couldn't resolve %s %s" % (remote_url, revision)) # Save to our cache self._git_ref_cache[remote_url, revision] = abs_revision p.setAttribute("revision", abs_revision) if abort: self.fatal("couldn't resolve some refs; exiting")
def resolve_refs(self, manifest): worker_pool = ThreadPool(20) lookup_threads_by_project = {} lookup_threads_by_parameters = {} # Resolve refnames for p in manifest.getElementsByTagName('project'): name = p.getAttribute('name') remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) # commit ids are already done if repo_manifest.is_commitid(revision): self.debug("%s is already locked to %s; skipping" % (name, revision)) continue # gaia is special - make sure we're using the same revision we used # for gaia.json if self.gaia_hg_revision and p.getAttribute('path') == 'gaia' and revision == self.config['gaia_git_branch']: git_rev = self.query_gaia_git_rev() self.info("Using %s for gaia to match %s in gaia.json" % (git_rev, self.gaia_hg_revision)) p.setAttribute('revision', git_rev) continue # If there's no '/' in the revision, assume it's a head if '/' not in revision: revision = 'refs/heads/%s' % revision cache_key = "%s:%s" % (remote_url, revision) # Check to see if we've looked up this revision on this remote # before. If we have, reuse the previous value rather than looking # it up again. This will make sure revisions for the same ref name # are consistent between devices, as long as they use the same # remote/refname. if cache_key in self._git_ref_cache: abs_revision = self._git_ref_cache[cache_key] self.info( "Reusing previous lookup %s -> %s" % (cache_key, abs_revision)) p.setAttribute('revision', abs_revision) continue # Maybe a thread already exists for this lookup, even if the result has not # yet been retrieved and placed in _git_ref_cache... # Please note result.get() can be called multiple times without problems; # the git command will only be executed once. Therefore we can associate many # projects to the same thread result, without problems later when we call # get() multiple times against the same thread result. if cache_key in lookup_threads_by_parameters: self.info("Reusing currently running thread to look up %s" % cache_key) lookup_threads_by_project[p] = lookup_threads_by_parameters.get(cache_key) else: async_result = worker_pool.apply_async(self.resolve_git_ref, (remote_url, revision)) lookup_threads_by_parameters[cache_key] = async_result lookup_threads_by_project[p] = async_result # TODO: alert/notify on missing repositories abort = False failed = [] for p, result in lookup_threads_by_project.iteritems(): abs_revision = result.get(timeout=300) remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) if not abs_revision: abort = True self.error("Couldn't resolve reference %s %s" % (remote_url, revision)) failed.append(p) p.setAttribute('revision', abs_revision) if abort: # Write message about how to set up syncing default = repo_manifest.get_default(manifest) for p in failed: if p.hasAttribute('remote'): remote = repo_manifest.get_remote(manifest, p.getAttribute('remote')) else: remote = repo_manifest.get_remote(manifest, default.getAttribute('remote')) new_fetch_url = remote.getAttribute('fetch') orig_fetch_url = self._remote_mappings[new_fetch_url] name = p.getAttribute('name') self.info("needs sync? %s/%s -> %s/%s" % (orig_fetch_url, name, new_fetch_url, name)) self.fatal("couldn't resolve some refs; exiting")