def __init__(self, options, builder, evaluator): """Constructor. Args: options: An argparse.Namespace to hold command line arguments. Should contain: * cros_flash_retry: Max retry for "cros flash" command. * cros_flash_sleep: #seconds to wait between retry. * cros_flash_backoff: backoff factor. Must be >=1. If backoff factor is 1, sleep_duration = sleep * num_retry. Otherwise, sleep_duration = sleep * (backoff_factor) ** (num_retry - 1) builder: Builder to build/deploy image. Should contain repo_dir. evaluator: Evaluator to get score """ super(ChromeOnCrosBisector, self).__init__(options, builder, evaluator) self.cros_flash_retry = max(0, options.cros_flash_retry) self.cros_flash_sleep = max(0, options.cros_flash_sleep) self.cros_flash_backoff = max(1, options.cros_flash_backoff) self.good_cros_version = None self.bad_cros_version = None self.bisect_between_cros_version = False if not (git.IsSHA1(options.good, full=False) and git.IsSHA1(options.bad, full=False)): # Postpone commit resolution to Run(). self.good_commit = None self.bad_commit = None self.good_cros_version = options.good self.bad_cros_version = options.bad self.bisect_between_cros_version = True # Used to access gs://. Lazy initialization. self.gs_ctx = None
def _FixUpManifests(self, repo_manifest): """Points the checkouts at the new branch in the manifests. Within the branch, make sure all manifests with projects that are "branchable" are checked out to "refs/heads/<new_branch>". Do this by updating all manifests in the known manifest projects. """ assert not self._run.options.delete_branch, 'Cannot fix a deleted branch.' # Use local branch ref. branch_ref = git.NormalizeRef(self.branch_name) logging.debug('Fixing manifest projects for new branch.') for project in site_config.params.MANIFEST_PROJECTS: manifest_checkout = repo_manifest.FindCheckout(project) manifest_dir = manifest_checkout['local_path'] push_remote = manifest_checkout['push_remote'] # Checkout revision can be either a sha1 or a branch ref. src_ref = manifest_checkout['revision'] if not git.IsSHA1(src_ref): src_ref = git.NormalizeRemoteRef(push_remote, src_ref) git.CreateBranch(manifest_dir, manifest_version.PUSH_BRANCH, src_ref) # We want to process default.xml and official.xml + their imports. pending_manifests = [ constants.DEFAULT_MANIFEST, constants.OFFICIAL_MANIFEST ] processed_manifests = [] while pending_manifests: # Canonicalize the manifest name (resolve dir and symlinks). manifest_path = os.path.join(manifest_dir, pending_manifests.pop()) manifest_path = os.path.realpath(manifest_path) # Don't process a manifest more than once. if manifest_path in processed_manifests: continue processed_manifests.append(manifest_path) if not os.path.exists(manifest_path): logging.info('Manifest not found: %s', manifest_path) continue logging.debug('Fixing manifest at %s.', manifest_path) included_manifests = self._UpdateManifest(manifest_path) pending_manifests += included_manifests git.RunGit(manifest_dir, ['add', '-A'], print_cmd=True) message = 'Fix up manifest after branching %s.' % branch_ref git.RunGit(manifest_dir, ['commit', '-m', message], print_cmd=True) push_to = git.RemoteRef(push_remote, branch_ref) git.GitPush(manifest_dir, manifest_version.PUSH_BRANCH, push_to, skip=self.skip_remote_push)
def _GetGerritPatch(self, query): """Query the configured helpers looking for a given change. Args: project: The gerrit project to query. query: A cros_patch.PatchQuery object. Returns: A GerritPatch object. """ helper = self._LookupHelper(query) query_text = query.ToGerritQueryText() change = helper.QuerySingleRecord( query_text, must_match=not git.IsSHA1(query_text)) if not change: return # If the query was a gerrit number based query, check the projects/change-id # to see if we already have it locally, but couldn't map it since we didn't # know the gerrit number at the time of the initial injection. existing = self._lookup_cache[change] if cros_patch.ParseGerritNumber(query_text) and existing is not None: keys = change.LookupAliases() self._lookup_cache.InjectCustomKeys(keys, existing) return existing self.InjectLookupCache([change]) if change.IsAlreadyMerged(): self.InjectCommittedPatches([change]) return change
def _GetGclientURLs(internal, rev): """Get the URLs and deps_file values to use in gclient file. See WriteConfigFile below. """ results = [] if rev is None or git.IsSHA1(rev): # Regular chromium checkout; src may float to origin/master or be pinned. url = constants.CHROMIUM_GOB_URL if rev: url += ('@' + rev) results.append(('src', url, '.DEPS.git')) if internal: results.append(('src-internal', constants.CHROME_INTERNAL_GOB_URL, '.DEPS.git')) elif internal: # Internal buildspec: check out the buildspec repo and set deps_file to # the path to the desired release spec. url = site_config.params.INTERNAL_GOB_URL + '/chrome/tools/buildspec.git' # Chromium switched to DEPS at version 45.0.2432.3. deps_file = '.DEPS.git' if BuildspecUsesDepsGit(rev) else 'DEPS' results.append( ('CHROME_DEPS', url, 'releases/%s/%s' % (rev, deps_file))) else: # External buildspec: use the main chromium src repository, pinned to the # release tag, with deps_file set to .DEPS.git (which is created by # publish_deps.py). url = constants.CHROMIUM_GOB_URL + '@refs/tags/' + rev results.append(('src', url, '.DEPS.git')) return results
def CheckCommitFormat(commit): """Checks if commit is the acceptable format. For git_bisector, commit should be SHA1. Child class may override it. Args: commit: commit string. Returns: Normalized commit. None if the format is unacceptable. """ if git.IsSHA1(commit, full=False): return commit return None
def main(argv): parser = GetParser() options = parser.parse_args(argv) repo_dir = git.FindRepoDir(os.getcwd()) if not repo_dir: parser.error("This script must be invoked from within a repository " "checkout.") options.git_config = os.path.join(repo_dir, 'manifests.git', 'config') options.local_manifest_path = os.path.join(repo_dir, 'local_manifest.xml') manifest_sym_path = os.path.join(repo_dir, 'manifest.xml') if os.path.basename(os.readlink(manifest_sym_path)) == 'minilayout.xml': _AssertNotMiniLayout() # For now, we only support the add command. assert options.command == 'add' if options.workon: if options.path is not None: parser.error('Adding workon projects do not set project.') else: if options.remote is None: parser.error('Adding non-workon projects requires a remote.') if options.path is None: parser.error('Adding non-workon projects requires a path.') name = options.project path = options.path revision = options.revision if revision is not None: if (not git.IsRefsTags(revision) and not git.IsSHA1(revision)): revision = git.StripRefsHeads(revision, False) main_manifest = git.ManifestCheckout(os.getcwd()) main_element = main_manifest.FindCheckouts(name) if path is not None: main_element_from_path = main_manifest.FindCheckoutFromPath( path, strict=False) if main_element_from_path is not None: main_element.append(main_element_from_path) local_manifest = LocalManifest.FromPath(options.local_manifest_path) if options.workon: if not main_element: parser.error('No project named %r in the default manifest.' % name) _AddProjectsToManifestGroups( options, [checkout['name'] for checkout in main_element]) elif main_element: if options.remote is not None: # Likely this project wasn't meant to be remote, so workon main element print( "Project already exists in manifest. Using that as workon project." ) _AddProjectsToManifestGroups( options, [checkout['name'] for checkout in main_element]) else: # Conflict will occur; complain. parser.error( "Requested project name=%r path=%r will conflict with " "your current manifest %s" % (name, path, main_manifest.manifest_path)) elif local_manifest.GetProject(name, path=path) is not None: parser.error("Requested project name=%r path=%r conflicts with " "your local_manifest.xml" % (name, path)) else: element = local_manifest.AddNonWorkonProject(name=name, path=path, remote=options.remote, revision=revision) _AddProjectsToManifestGroups(options, [element.attrib['name']]) with open(options.local_manifest_path, 'w') as f: f.write(local_manifest.ToString()) return 0
def _ProcessCheckout(self, src_manifest, src_checkout): """Performs per-checkout push operations. Args: src_manifest: The ManifestCheckout object for the current manifest. src_checkout: The ProjectCheckout object to process. """ if not src_checkout.IsBranchableProject(): # We don't have the ability to push branches to this repository. Just # use TOT instead. return checkout_name = src_checkout['name'] remote = src_checkout['push_remote'] src_ref = src_checkout['revision'] suffix = self._GetBranchSuffix(src_manifest, src_checkout) # The source/destination branches depend on options. if self.rename_to: # Rename flow. Both src and dst branches exist. src_branch = '%s%s' % (self.branch_name, suffix) dst_branch = '%s%s' % (self.rename_to, suffix) elif self._run.options.delete_branch: # Delete flow. Only dst branch exists. src_branch = None dst_branch = '%s%s' % (self.branch_name, suffix) else: # Create flow (default). Only dst branch exists. Source # for the branch will just be src_ref. src_branch = None dst_branch = '%s%s' % (self.branch_name, suffix) # Normalize branch refs to remote. We only process remote branches. src_branch = git.NormalizeRemoteRef(remote, src_branch) dst_branch = git.NormalizeRemoteRef(remote, dst_branch) # Determine whether src/dst branches exist now, by getting their sha1s. if src_branch: src_sha1 = self._GetSHA1(src_checkout, src_branch) elif git.IsSHA1(src_ref): src_sha1 = src_ref dst_sha1 = self._GetSHA1(src_checkout, dst_branch) # Complain if the branch already exists, unless that is expected. force = self._run.options.force_create or self._run.options.delete_branch if dst_sha1 and not force: # We are either creating a branch or renaming a branch, and the # destination branch unexpectedly exists. Accept this only if the # destination branch is already at the revision we want. if src_sha1 != dst_sha1: raise BranchError( 'Checkout %s already contains branch %s. Run with ' '--force-create to overwrite.' % (checkout_name, dst_branch)) logging.info( 'Checkout %s already contains branch %s and it already' ' points to revision %s', checkout_name, dst_branch, dst_sha1) elif self._run.options.delete_branch: # Delete the dst_branch, if it exists. if dst_sha1: self._DeleteBranch(src_checkout, dst_branch) else: raise BranchError( 'Checkout %s does not contain branch %s to delete.' % (checkout_name, dst_branch)) elif self.rename_to: # Copy src_branch to dst_branch, if it exists, then delete src_branch. if src_sha1: self._CopyBranch(src_checkout, src_branch, dst_branch) self._DeleteBranch(src_checkout, src_branch) else: raise BranchError( 'Checkout %s does not contain branch %s to rename.' % (checkout_name, src_branch)) else: # Copy src_ref to dst_branch. self._CopyBranch(src_checkout, src_ref, dst_branch, force=self._run.options.force_create)
def main(argv): parser = optparse.OptionParser(usage='usage: %prog add [options] <name> ' '<--workon | <path> --remote <remote> >') parser.add_option('-w', '--workon', action='store_true', dest='workon', default=False, help='Is this a workon package?') parser.add_option('-r', '--remote', dest='remote', default=None) parser.add_option('-v', '--revision', dest='revision', default=None, help="Use to override the manifest defined default " "revision used for a given project.") parser.add_option('--upgrade-minilayout', default=False, action='store_true', help="Upgrade a minilayout checkout into a full.xml " "checkout utilizing manifest groups.") (options, args) = parser.parse_args(argv) repo_dir = git.FindRepoDir(os.getcwd()) if not repo_dir: parser.error("This script must be invoked from within a repository " "checkout.") options.git_config = os.path.join(repo_dir, 'manifests.git', 'config') options.repo_dir = repo_dir options.local_manifest_path = os.path.join(repo_dir, 'local_manifest.xml') # This constant is used only when we're doing an upgrade away from # minilayout.xml to default.xml. options.default_manifest_path = os.path.join(repo_dir, 'manifests', 'default.xml') options.manifest_sym_path = os.path.join(repo_dir, 'manifest.xml') active_manifest = os.path.basename(os.readlink(options.manifest_sym_path)) upgrade_required = active_manifest == 'minilayout.xml' if options.upgrade_minilayout: if args: parser.error("--upgrade-minilayout takes no arguments.") if not upgrade_required: print "This repository checkout isn't using minilayout.xml; nothing to do" else: _UpgradeMinilayout(options) return 0 elif upgrade_required: logging.warn( "Your repository checkout is using the old minilayout.xml workflow; " "auto-upgrading it.") cros_build_lib.RunCommand([sys.argv[0], '--upgrade-minilayout'], cwd=os.getcwd(), print_cmd=False) if not args: parser.error("No command specified.") elif args[0] != 'add': parser.error("Only supported subcommand is add right now.") elif options.workon: if len(args) != 2: parser.error( "Argument count is wrong for --workon; must be add <project>") name, path = args[1], None else: if options.remote is None: parser.error('Adding non-workon projects requires a remote.') elif len(args) != 3: parser.error("Argument count is wrong for non-workon mode; " "must be add <project> <path> --remote <remote-arg>") name, path = args[1:] revision = options.revision if revision is not None: if (not git.IsRefsTags(revision) and not git.IsSHA1(revision)): revision = git.StripRefsHeads(revision, False) main_manifest = Manifest.FromPath(options.manifest_sym_path, empty_if_missing=False) local_manifest = Manifest.FromPath(options.local_manifest_path) main_element = main_manifest.GetProject(name, path=path) if options.workon: if main_element is None: parser.error('No project named %r in the default manifest.' % name) _AddProjectsToManifestGroups(options, main_element.attrib['name']) elif main_element is not None: if options.remote is not None: # Likely this project wasn't meant to be remote, so workon main element print "Project already exists in manifest. Using that as workon project." _AddProjectsToManifestGroups(options, main_element.attrib['name']) else: # Conflict will occur; complain. parser.error( "Requested project name=%r path=%r will conflict with " "your current manifest %s" % (name, path, active_manifest)) elif local_manifest.GetProject(name, path=path) is not None: parser.error("Requested project name=%r path=%r conflicts with " "your local_manifest.xml" % (name, path)) else: element = local_manifest.AddNonWorkonProject(name=name, path=path, remote=options.remote, revision=revision) _AddProjectsToManifestGroups(options, element.attrib['name']) with open(options.local_manifest_path, 'w') as f: f.write(local_manifest.ToString()) return 0
def main(argv): parser = GetParser() options = parser.parse_args(argv) repo_dir = git.FindRepoDir(os.getcwd()) if not repo_dir: parser.error("This script must be invoked from within a repository " "checkout.") options.git_config = os.path.join(repo_dir, 'manifests.git', 'config') options.repo_dir = repo_dir options.local_manifest_path = os.path.join(repo_dir, 'local_manifest.xml') # This constant is used only when we're doing an upgrade away from # minilayout.xml to default.xml. options.default_manifest_path = os.path.join(repo_dir, 'manifests', 'default.xml') options.manifest_sym_path = os.path.join(repo_dir, 'manifest.xml') active_manifest = os.path.basename(os.readlink(options.manifest_sym_path)) upgrade_required = active_manifest == 'minilayout.xml' if options.command == 'upgrade-minilayout': if not upgrade_required: print("This repository checkout isn't using minilayout.xml; " "nothing to do") else: _UpgradeMinilayout(options) return 0 elif upgrade_required: logging.warning( "Your repository checkout is using the old minilayout.xml workflow; " "auto-upgrading it.") main(['upgrade-minilayout']) # For now, we only support the add command. assert options.command == 'add' if options.workon: if options.path is not None: parser.error('Adding workon projects do not set project.') else: if options.remote is None: parser.error('Adding non-workon projects requires a remote.') if options.path is None: parser.error('Adding non-workon projects requires a path.') name = options.project path = options.path revision = options.revision if revision is not None: if (not git.IsRefsTags(revision) and not git.IsSHA1(revision)): revision = git.StripRefsHeads(revision, False) main_manifest = Manifest.FromPath(options.manifest_sym_path, empty_if_missing=False) local_manifest = Manifest.FromPath(options.local_manifest_path) main_element = main_manifest.GetProject(name, path=path) if options.workon: if main_element is None: parser.error('No project named %r in the default manifest.' % name) _AddProjectsToManifestGroups(options, main_element.attrib['name']) elif main_element is not None: if options.remote is not None: # Likely this project wasn't meant to be remote, so workon main element print( "Project already exists in manifest. Using that as workon project." ) _AddProjectsToManifestGroups(options, main_element.attrib['name']) else: # Conflict will occur; complain. parser.error( "Requested project name=%r path=%r will conflict with " "your current manifest %s" % (name, path, active_manifest)) elif local_manifest.GetProject(name, path=path) is not None: parser.error("Requested project name=%r path=%r conflicts with " "your local_manifest.xml" % (name, path)) else: element = local_manifest.AddNonWorkonProject(name=name, path=path, remote=options.remote, revision=revision) _AddProjectsToManifestGroups(options, element.attrib['name']) with open(options.local_manifest_path, 'w') as f: f.write(local_manifest.ToString()) return 0