def checkout_sources(self): dirs = self.query_abs_dirs() gecko_config = self.load_gecko_config() b2g_manifest_intree = gecko_config.get('b2g_manifest_intree') if gecko_config.get('config_version') >= 2: repos = [ {'vcs': 'gittool', 'repo': 'https://git.mozilla.org/b2g/B2G.git', 'dest': dirs['work_dir']}, ] if b2g_manifest_intree: # Checkout top-level B2G repo now self.vcs_checkout_repos(repos) b2g_manifest_branch = 'master' # That may have blown away our build-tools checkout. It would # be better if B2G were checked out into a subdirectory, but # for now, just redo it. self.checkout_tools() # Now checkout gecko inside the build directory self.checkout_gecko() conf_dir = os.path.join(dirs['gecko_src'], os.path.dirname(self.query_gecko_config_path())) manifest_filename = os.path.join(conf_dir, 'sources.xml') self.info("Using manifest at %s" % manifest_filename) have_gecko = True else: # Checkout B2G and b2g-manifests. We'll do gecko later b2g_manifest_branch = gecko_config.get('b2g_manifest_branch', 'master') repos.append( {'vcs': 'gittool', 'repo': 'https://git.mozilla.org/b2g/b2g-manifest.git', 'dest': os.path.join(dirs['work_dir'], 'b2g-manifest'), 'branch': b2g_manifest_branch}, ) manifest_filename = gecko_config.get('b2g_manifest', self.config['target'] + '.xml') manifest_filename = os.path.join(dirs['work_dir'], 'b2g-manifest', manifest_filename) self.vcs_checkout_repos(repos) have_gecko = False manifest = load_manifest(manifest_filename) if not b2g_manifest_intree: # Now munge the manifest by mapping remotes to local remotes mapping_func = functools.partial(map_remote, mappings=self.config['repo_remote_mappings']) rewrite_remotes(manifest, mapping_func) # Remove gecko, since we'll be checking that out ourselves gecko_node = remove_project(manifest, path='gecko') if not gecko_node: self.fatal("couldn't remove gecko from manifest") # Write out our manifest locally manifest_dir = os.path.join(dirs['work_dir'], 'tmp_manifest') self.rmtree(manifest_dir) self.mkdir_p(manifest_dir) manifest_filename = os.path.join(manifest_dir, self.config['target'] + '.xml') self.info("Writing manifest to %s" % manifest_filename) manifest_file = open(manifest_filename, 'w') manifest.writexml(manifest_file) manifest_file.close() # Set up repo repo_link = os.path.join(dirs['work_dir'], '.repo') if 'repo_mirror_dir' in self.config: # Make our local .repo directory a symlink to the shared repo # directory repo_mirror_dir = self.config['repo_mirror_dir'] self.mkdir_p(repo_mirror_dir) repo_link = os.path.join(dirs['work_dir'], '.repo') if not os.path.exists(repo_link) or not os.path.islink(repo_link): self.rmtree(repo_link) self.info("Creating link from %s to %s" % (repo_link, repo_mirror_dir)) os.symlink(repo_mirror_dir, repo_link) # Checkout the repo tool if 'repo_repo' in self.config: repo_dir = os.path.join(dirs['work_dir'], '.repo', 'repo') self.checkout_repotool(repo_dir) cmd = ['./repo', '--version'] if not self.run_command(cmd, cwd=dirs['work_dir']) == 0: # Set return code to RETRY self.fatal("repo is broken", exit_code=4) # Check it out! max_tries = 5 sleep_time = 60 max_sleep_time = 300 for _ in range(max_tries): # If .repo points somewhere, then try and reset our state # before running config.sh if os.path.isdir(repo_link): # Delete any projects with broken HEAD references self.info("Deleting broken projects...") cmd = ['./repo', 'forall', '-c', 'git show-ref -q --head HEAD || rm -rfv $PWD'] self.run_command(cmd, cwd=dirs['work_dir']) config_result = self.run_command([ './config.sh', '-q', self.config['target'], manifest_filename, ], cwd=dirs['work_dir'], output_timeout=55 * 60) # TODO: Check return code from these? retry? # Run git reset --hard to make sure we're in a clean state self.info("Resetting all git projects") cmd = ['./repo', 'forall', '-c', 'git reset --hard'] self.run_command(cmd, cwd=dirs['work_dir']) self.info("Cleaning all git projects") cmd = ['./repo', 'forall', '-c', 'git clean -f -x -d'] self.run_command(cmd, cwd=dirs['work_dir']) if config_result == 0: break else: # We may have died due to left-over lock files. Make sure # we clean those up before trying again. self.info("Deleting stale lock files") cmd = ['find', '.repo/', '-name', '*.lock', '-print', '-delete'] self.run_command(cmd, cwd=dirs['work_dir']) # Try again in a bit. Broken clones should be deleted and # re-tried above self.info("config.sh failed; sleeping %i and retrying" % sleep_time) time.sleep(sleep_time) # Exponential backoff with random jitter sleep_time = min(sleep_time * 1.5, max_sleep_time) + random.randint(1, 60) else: self.fatal("failed to run config.sh") # Workaround bug 985837 if self.config['target'] == 'emulator-kk': self.info("Forcing -j4 for emulator-kk") dotconfig_file = os.path.join(dirs['abs_work_dir'], '.config') with open(dotconfig_file, "a+") as f: f.write("\nMAKE_FLAGS=-j1\n") # output our sources.xml, make a copy for update_sources_xml() self.run_command( ["./gonk-misc/add-revision.py", "-o", "sources.xml", "--force", ".repo/manifest.xml"], cwd=dirs["work_dir"], halt_on_failure=True, fatal_exit_code=3) self.run_command(["cat", "sources.xml"], cwd=dirs['work_dir'], halt_on_failure=True, fatal_exit_code=3) self.run_command(["cp", "-p", "sources.xml", "sources.xml.original"], cwd=dirs['work_dir'], halt_on_failure=True, fatal_exit_code=3) manifest = load_manifest(os.path.join(dirs['work_dir'], 'sources.xml')) gaia_node = get_project(manifest, path="gaia") gaia_rev = gaia_node.getAttribute("revision") gaia_remote = get_remote(manifest, gaia_node.getAttribute('remote')) gaia_repo = "%s/%s" % (gaia_remote.getAttribute('fetch'), gaia_node.getAttribute('name')) gaia_url = self.query_gitweb_url(gaia_repo, gaia_rev) self.set_buildbot_property("gaia_revision", gaia_rev, write_to_file=True) self.info("TinderboxPrint: gaia_revlink: %s" % gaia_url) # Now we can checkout gecko and other stuff if not have_gecko: self.checkout_gecko() return # Old behaviour self.checkout_gecko() self.checkout_gaia()
def resolve_refs(self, manifest): worker_pool = ThreadPool(20) lookup_threads_by_project = {} lookup_threads_by_parameters = {} # Resolve refnames for p in manifest.getElementsByTagName('project'): name = p.getAttribute('name') remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) # commit ids are already done if repo_manifest.is_commitid(revision): self.debug("%s is already locked to %s; skipping" % (name, revision)) continue # gaia is special - make sure we're using the same revision we used # for gaia.json if self.gaia_hg_revision and p.getAttribute('path') == 'gaia' and revision == self.config['gaia_git_branch']: git_rev = self.query_gaia_git_rev() self.info("Using %s for gaia to match %s in gaia.json" % (git_rev, self.gaia_hg_revision)) p.setAttribute('revision', git_rev) continue # If there's no '/' in the revision, assume it's a head if '/' not in revision: revision = 'refs/heads/%s' % revision cache_key = "%s:%s" % (remote_url, revision) # Check to see if we've looked up this revision on this remote # before. If we have, reuse the previous value rather than looking # it up again. This will make sure revisions for the same ref name # are consistent between devices, as long as they use the same # remote/refname. if cache_key in self._git_ref_cache: abs_revision = self._git_ref_cache[cache_key] self.debug( "Reusing previous lookup %s -> %s" % (cache_key, abs_revision)) p.setAttribute('revision', abs_revision) continue # Maybe a thread already exists for this lookup, even if the result has not # yet been retrieved and placed in _git_ref_cache... # Please note result.get() can be called multiple times without problems; # the git command will only be executed once. Therefore we can associate many # projects to the same thread result, without problems later when we call # get() multiple times against the same thread result. if cache_key in lookup_threads_by_parameters: self.debug("Reusing currently running thread to look up %s" % cache_key) lookup_threads_by_project[p] = lookup_threads_by_parameters.get(cache_key) else: async_result = worker_pool.apply_async(self.resolve_git_ref, (remote_url, revision)) lookup_threads_by_parameters[cache_key] = async_result lookup_threads_by_project[p] = async_result # TODO: alert/notify on missing repositories abort = False failed = [] for p, result in lookup_threads_by_project.iteritems(): abs_revision = result.get(timeout=300) remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) if not abs_revision: abort = True self.error("Couldn't resolve reference %s %s" % (remote_url, revision)) failed.append(p) p.setAttribute('revision', abs_revision) if abort: # Write message about how to set up syncing default = repo_manifest.get_default(manifest) for p in failed: if p.hasAttribute('remote'): remote = repo_manifest.get_remote(manifest, p.getAttribute('remote')) else: remote = repo_manifest.get_remote(manifest, default.getAttribute('remote')) new_fetch_url = remote.getAttribute('fetch') orig_fetch_url = self._remote_mappings[new_fetch_url] name = p.getAttribute('name') self.info("needs sync? %s/%s -> %s/%s" % (orig_fetch_url, name, new_fetch_url, name)) self.fatal("couldn't resolve some refs; exiting")
def resolve_refs(self, manifest): worker_pool = ThreadPool(20) results = [] # Resolve refnames for p in manifest.getElementsByTagName('project'): name = p.getAttribute('name') remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) # commit ids are already done if repo_manifest.is_commitid(revision): self.debug("%s is already locked to %s; skipping" % (name, revision)) continue # gaia is special - make sure we're using the same revision we used # for gaia.json if self.gaia_hg_revision and p.getAttribute('path') == 'gaia' and revision == self.config['gaia_git_branch']: if not self.gaia_git_rev: self.gaia_git_rev = self.query_mapper_git_revision( self.config['mapper_url'], self.config['gaia_mapper_project'], self.gaia_hg_revision, ) self.info("Using %s for gaia to match %s in gaia.json" % (self.gaia_git_rev, self.gaia_hg_revision)) p.setAttribute('revision', self.gaia_git_rev) continue # Check to see if we've looked up this revision on this remote # before If we have, reuse the previous value rather than looking # it up again This will make sure revisions for the same ref name # are consistent between devices, as long as they use the same # remote/refname if (remote_url, revision) in self._git_ref_cache: abs_revision = self._git_ref_cache[remote_url, revision] self.info( "Re-using previous lookup %s:%s -> %s" % (remote_url, revision, abs_revision)) p.setAttribute('revision', abs_revision) continue # If there's no '/' in the revision, assume it's a head if '/' not in revision: revision = 'refs/heads/%s' % revision self.debug("Getting revision for %s (currently %s)" % (name, revision)) async_result = worker_pool.apply_async(self.resolve_git_ref, (remote_url, revision)) results.append((p, async_result)) # TODO: alert/notify on missing repositories # TODO: Add external caching abort = False failed = [] for p, result in results: abs_revision = result.get() remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) if not abs_revision: abort = True self.error("Couldn't resolve %s %s" % (remote_url, revision)) failed.append(p) # Save to our cache self._git_ref_cache[remote_url, revision] = abs_revision p.setAttribute('revision', abs_revision) if abort: # Write message about how to set up syncing default = repo_manifest.get_default(manifest) for p in failed: if p.hasAttribute('remote'): remote = repo_manifest.get_remote(manifest, p.getAttribute('remote')) else: remote = repo_manifest.get_remote(manifest, default.getAttribute('remote')) new_fetch_url = remote.getAttribute('fetch') orig_fetch_url = self._remote_mappings[new_fetch_url] name = p.getAttribute('name') self.info("needs sync? %s/%s -> %s/%s" % (orig_fetch_url, name, new_fetch_url, name)) self.fatal("couldn't resolve some refs; exiting")
def checkout_sources(self): dirs = self.query_abs_dirs() gecko_config = self.load_gecko_config() b2g_manifest_intree = gecko_config.get('b2g_manifest_intree') b2g_repo = gecko_config.get('b2g_repo', 'https://github.com/mozilla-b2g/B2G.git') b2g_branch = gecko_config.get('b2g_branch', 'master') if gecko_config.get('config_version') >= 2: repos = [ { 'vcs': 'gittool', 'repo': b2g_repo, 'branch': b2g_branch, 'dest': dirs['work_dir'] }, ] if b2g_manifest_intree: # Checkout top-level B2G repo now self.vcs_checkout_repos(repos) b2g_manifest_branch = 'master' # That may have blown away our build-tools checkout. It would # be better if B2G were checked out into a subdirectory, but # for now, just redo it. self.checkout_tools() # Now checkout gecko inside the build directory self.checkout_gecko() conf_dir = os.path.join( dirs['gecko_src'], os.path.dirname(self.query_gecko_config_path())) manifest_filename = os.path.join(conf_dir, 'sources.xml') self.info("Using manifest at %s" % manifest_filename) have_gecko = True else: # Checkout B2G and b2g-manifests. We'll do gecko later b2g_manifest_branch = gecko_config.get('b2g_manifest_branch', 'master') repos.append( { 'vcs': 'gittool', 'repo': 'https://github.com/mozilla-b2g/b2g-manifest', 'dest': os.path.join(dirs['work_dir'], 'b2g-manifest'), 'branch': b2g_manifest_branch }, ) manifest_filename = gecko_config.get( 'b2g_manifest', self.config['target'] + '.xml') manifest_filename = os.path.join(dirs['work_dir'], 'b2g-manifest', manifest_filename) self.vcs_checkout_repos(repos) have_gecko = False manifest = load_manifest(manifest_filename) if not b2g_manifest_intree: # Now munge the manifest by mapping remotes to local remotes mapping_func = functools.partial( map_remote, mappings=self.config['repo_remote_mappings']) rewrite_remotes(manifest, mapping_func) # Remove gecko, since we'll be checking that out ourselves gecko_node = remove_project(manifest, path='gecko') if not gecko_node: self.fatal("couldn't remove gecko from manifest") # Write out our manifest locally manifest_dir = os.path.join(dirs['work_dir'], 'tmp_manifest') self.rmtree(manifest_dir) self.mkdir_p(manifest_dir) manifest_filename = os.path.join(manifest_dir, self.config['target'] + '.xml') self.info("Writing manifest to %s" % manifest_filename) manifest_file = open(manifest_filename, 'w') manifest.writexml(manifest_file) manifest_file.close() # Set up repo repo_link = os.path.join(dirs['work_dir'], '.repo') if 'repo_mirror_dir' in self.config: # Make our local .repo directory a symlink to the shared repo # directory repo_mirror_dir = self.config['repo_mirror_dir'] self.mkdir_p(repo_mirror_dir) repo_link = os.path.join(dirs['work_dir'], '.repo') if not os.path.exists(repo_link) or not os.path.islink( repo_link): self.rmtree(repo_link) self.info("Creating link from %s to %s" % (repo_link, repo_mirror_dir)) os.symlink(repo_mirror_dir, repo_link) # Checkout the repo tool if 'repo_repo' in self.config: repo_dir = os.path.join(dirs['work_dir'], '.repo', 'repo') self.checkout_repotool(repo_dir) cmd = ['./repo', '--version'] if not self.run_command(cmd, cwd=dirs['work_dir']) == 0: # Set return code to RETRY self.fatal("repo is broken", exit_code=4) # Check it out! max_tries = 5 sleep_time = 60 max_sleep_time = 300 for _ in range(max_tries): # If .repo points somewhere, then try and reset our state # before running config.sh if os.path.isdir(repo_link): # Delete any projects with broken HEAD references self.info("Deleting broken projects...") cmd = [ './repo', 'forall', '-c', 'git show-ref -q --head HEAD || rm -rfv $PWD' ] self.run_command(cmd, cwd=dirs['work_dir']) # bug https://bugzil.la/1177190 - workaround - change # timeout from 55 to 10 min, based on "normal" being # about 7.5 minutes config_result = self.run_command([ './config.sh', '-q', self.config['target'], manifest_filename, ], cwd=dirs['work_dir'], output_timeout=10 * 60) # TODO: Check return code from these? retry? # Run git reset --hard to make sure we're in a clean state self.info("Resetting all git projects") cmd = ['./repo', 'forall', '-c', 'git reset --hard'] self.run_command(cmd, cwd=dirs['work_dir']) self.info("Cleaning all git projects") cmd = ['./repo', 'forall', '-c', 'git clean -f -x -d'] self.run_command(cmd, cwd=dirs['work_dir']) if config_result == 0: break else: # We may have died due to left-over lock files. Make sure # we clean those up before trying again. self.info("Deleting stale lock files") cmd = [ 'find', '.repo/', '-name', '*.lock', '-print', '-delete' ] self.run_command(cmd, cwd=dirs['work_dir']) # Try again in a bit. Broken clones should be deleted and # re-tried above self.info("config.sh failed; sleeping %i and retrying" % sleep_time) time.sleep(sleep_time) # Exponential backoff with random jitter sleep_time = min(sleep_time * 1.5, max_sleep_time) + random.randint(1, 60) else: self.fatal("failed to run config.sh") # output our sources.xml, make a copy for update_sources_xml() self.run_command([ "./gonk-misc/add-revision.py", "-o", "sources.xml", "--force", ".repo/manifest.xml" ], cwd=dirs["work_dir"], halt_on_failure=True, fatal_exit_code=3) self.run_command(["cat", "sources.xml"], cwd=dirs['work_dir'], halt_on_failure=True, fatal_exit_code=3) self.run_command( ["cp", "-p", "sources.xml", "sources.xml.original"], cwd=dirs['work_dir'], halt_on_failure=True, fatal_exit_code=3) manifest = load_manifest( os.path.join(dirs['work_dir'], 'sources.xml')) gaia_node = get_project(manifest, path="gaia") gaia_rev = gaia_node.getAttribute("revision") gaia_remote = get_remote(manifest, gaia_node.getAttribute('remote')) gaia_repo = "%s/%s" % (gaia_remote.getAttribute('fetch'), gaia_node.getAttribute('name')) gaia_url = self.query_gitweb_url(gaia_repo, gaia_rev) self.set_buildbot_property("gaia_revision", gaia_rev, write_to_file=True) self.info("TinderboxPrint: gaia_revlink: %s" % gaia_url) # Now we can checkout gecko and other stuff if not have_gecko: self.checkout_gecko() return # Old behaviour self.checkout_gecko() self.checkout_gaia()
def resolve_refs(self, manifest): worker_pool = ThreadPool(20) lookup_threads_by_project = {} lookup_threads_by_parameters = {} # Resolve refnames for p in manifest.getElementsByTagName('project'): name = p.getAttribute('name') remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) # commit ids are already done if repo_manifest.is_commitid(revision): self.debug("%s is already locked to %s; skipping" % (name, revision)) continue # gaia is special - make sure we're using the same revision we used # for gaia.json if self.gaia_hg_revision and p.getAttribute('path') == 'gaia' and revision == self.config['gaia_git_branch']: git_rev = self.query_gaia_git_rev() self.info("Using %s for gaia to match %s in gaia.json" % (git_rev, self.gaia_hg_revision)) p.setAttribute('revision', git_rev) continue # If there's no '/' in the revision, assume it's a head if '/' not in revision: revision = 'refs/heads/%s' % revision cache_key = "%s:%s" % (remote_url, revision) # Check to see if we've looked up this revision on this remote # before. If we have, reuse the previous value rather than looking # it up again. This will make sure revisions for the same ref name # are consistent between devices, as long as they use the same # remote/refname. if cache_key in self._git_ref_cache: abs_revision = self._git_ref_cache[cache_key] self.info( "Reusing previous lookup %s -> %s" % (cache_key, abs_revision)) p.setAttribute('revision', abs_revision) continue # Maybe a thread already exists for this lookup, even if the result has not # yet been retrieved and placed in _git_ref_cache... # Please note result.get() can be called multiple times without problems; # the git command will only be executed once. Therefore we can associate many # projects to the same thread result, without problems later when we call # get() multiple times against the same thread result. if cache_key in lookup_threads_by_parameters: self.info("Reusing currently running thread to look up %s" % cache_key) lookup_threads_by_project[p] = lookup_threads_by_parameters.get(cache_key) else: async_result = worker_pool.apply_async(self.resolve_git_ref, (remote_url, revision)) lookup_threads_by_parameters[cache_key] = async_result lookup_threads_by_project[p] = async_result # TODO: alert/notify on missing repositories abort = False failed = [] for p, result in lookup_threads_by_project.iteritems(): abs_revision = result.get(timeout=300) remote_url = repo_manifest.get_project_remote_url(manifest, p) revision = repo_manifest.get_project_revision(manifest, p) if not abs_revision: abort = True self.error("Couldn't resolve reference %s %s" % (remote_url, revision)) failed.append(p) p.setAttribute('revision', abs_revision) if abort: # Write message about how to set up syncing default = repo_manifest.get_default(manifest) for p in failed: if p.hasAttribute('remote'): remote = repo_manifest.get_remote(manifest, p.getAttribute('remote')) else: remote = repo_manifest.get_remote(manifest, default.getAttribute('remote')) new_fetch_url = remote.getAttribute('fetch') orig_fetch_url = self._remote_mappings[new_fetch_url] name = p.getAttribute('name') self.info("needs sync? %s/%s -> %s/%s" % (orig_fetch_url, name, new_fetch_url, name)) self.fatal("couldn't resolve some refs; exiting")