def __init__(self, options, builder, evaluator):
"""Constructor.
Args:
options: An argparse.Namespace to hold command line arguments. Should
contain:
* cros_flash_retry: Max retry for "cros flash" command.
* cros_flash_sleep: #seconds to wait between retry.
* cros_flash_backoff: backoff factor. Must be >=1. If backoff factor
is 1, sleep_duration = sleep * num_retry. Otherwise,
sleep_duration = sleep * (backoff_factor) ** (num_retry - 1)
builder: Builder to build/deploy image. Should contain repo_dir.
evaluator: Evaluator to get score
"""
super(ChromeOnCrosBisector, self).__init__(options, builder, evaluator)
self.cros_flash_retry = max(0, options.cros_flash_retry)
self.cros_flash_sleep = max(0, options.cros_flash_sleep)
self.cros_flash_backoff = max(1, options.cros_flash_backoff)
self.good_cros_version = None
self.bad_cros_version = None
self.bisect_between_cros_version = False
if not (git.IsSHA1(options.good, full=False)
and git.IsSHA1(options.bad, full=False)):
# Postpone commit resolution to Run().
self.good_commit = None
self.bad_commit = None
self.good_cros_version = options.good
self.bad_cros_version = options.bad
self.bisect_between_cros_version = True
# Used to access gs://. Lazy initialization.
self.gs_ctx = None
def _FixUpManifests(self, repo_manifest):
"""Points the checkouts at the new branch in the manifests.
Within the branch, make sure all manifests with projects that are
"branchable" are checked out to "refs/heads/<new_branch>". Do this
by updating all manifests in the known manifest projects.
"""
assert not self._run.options.delete_branch, 'Cannot fix a deleted branch.'
# Use local branch ref.
branch_ref = git.NormalizeRef(self.branch_name)
logging.debug('Fixing manifest projects for new branch.')
for project in site_config.params.MANIFEST_PROJECTS:
manifest_checkout = repo_manifest.FindCheckout(project)
manifest_dir = manifest_checkout['local_path']
push_remote = manifest_checkout['push_remote']
# Checkout revision can be either a sha1 or a branch ref.
src_ref = manifest_checkout['revision']
if not git.IsSHA1(src_ref):
src_ref = git.NormalizeRemoteRef(push_remote, src_ref)
git.CreateBranch(manifest_dir, manifest_version.PUSH_BRANCH,
src_ref)
# We want to process default.xml and official.xml + their imports.
pending_manifests = [
constants.DEFAULT_MANIFEST, constants.OFFICIAL_MANIFEST
]
processed_manifests = []
while pending_manifests:
# Canonicalize the manifest name (resolve dir and symlinks).
manifest_path = os.path.join(manifest_dir,
pending_manifests.pop())
manifest_path = os.path.realpath(manifest_path)
# Don't process a manifest more than once.
if manifest_path in processed_manifests:
continue
processed_manifests.append(manifest_path)
if not os.path.exists(manifest_path):
logging.info('Manifest not found: %s', manifest_path)
continue
logging.debug('Fixing manifest at %s.', manifest_path)
included_manifests = self._UpdateManifest(manifest_path)
pending_manifests += included_manifests
git.RunGit(manifest_dir, ['add', '-A'], print_cmd=True)
message = 'Fix up manifest after branching %s.' % branch_ref
git.RunGit(manifest_dir, ['commit', '-m', message], print_cmd=True)
push_to = git.RemoteRef(push_remote, branch_ref)
git.GitPush(manifest_dir,
manifest_version.PUSH_BRANCH,
push_to,
skip=self.skip_remote_push)
def _GetGerritPatch(self, query):
"""Query the configured helpers looking for a given change.
Args:
project: The gerrit project to query.
query: A cros_patch.PatchQuery object.
Returns:
A GerritPatch object.
"""
helper = self._LookupHelper(query)
query_text = query.ToGerritQueryText()
change = helper.QuerySingleRecord(
query_text, must_match=not git.IsSHA1(query_text))
if not change:
return
# If the query was a gerrit number based query, check the projects/change-id
# to see if we already have it locally, but couldn't map it since we didn't
# know the gerrit number at the time of the initial injection.
existing = self._lookup_cache[change]
if cros_patch.ParseGerritNumber(query_text) and existing is not None:
keys = change.LookupAliases()
self._lookup_cache.InjectCustomKeys(keys, existing)
return existing
self.InjectLookupCache([change])
if change.IsAlreadyMerged():
self.InjectCommittedPatches([change])
return change
def _GetGclientURLs(internal, rev):
"""Get the URLs and deps_file values to use in gclient file.
See WriteConfigFile below.
"""
results = []
if rev is None or git.IsSHA1(rev):
# Regular chromium checkout; src may float to origin/master or be pinned.
url = constants.CHROMIUM_GOB_URL
if rev:
url += ('@' + rev)
results.append(('src', url, '.DEPS.git'))
if internal:
results.append(('src-internal', constants.CHROME_INTERNAL_GOB_URL,
'.DEPS.git'))
elif internal:
# Internal buildspec: check out the buildspec repo and set deps_file to
# the path to the desired release spec.
url = site_config.params.INTERNAL_GOB_URL + '/chrome/tools/buildspec.git'
# Chromium switched to DEPS at version 45.0.2432.3.
deps_file = '.DEPS.git' if BuildspecUsesDepsGit(rev) else 'DEPS'
results.append(
('CHROME_DEPS', url, 'releases/%s/%s' % (rev, deps_file)))
else:
# External buildspec: use the main chromium src repository, pinned to the
# release tag, with deps_file set to .DEPS.git (which is created by
# publish_deps.py).
url = constants.CHROMIUM_GOB_URL + '@refs/tags/' + rev
results.append(('src', url, '.DEPS.git'))
return results
def CheckCommitFormat(commit):
"""Checks if commit is the acceptable format.
For git_bisector, commit should be SHA1. Child class may override it.
Args:
commit: commit string.
Returns:
Normalized commit. None if the format is unacceptable.
"""
if git.IsSHA1(commit, full=False):
return commit
return None
def main(argv):
parser = GetParser()
options = parser.parse_args(argv)
repo_dir = git.FindRepoDir(os.getcwd())
if not repo_dir:
parser.error("This script must be invoked from within a repository "
"checkout.")
options.git_config = os.path.join(repo_dir, 'manifests.git', 'config')
options.local_manifest_path = os.path.join(repo_dir, 'local_manifest.xml')
manifest_sym_path = os.path.join(repo_dir, 'manifest.xml')
if os.path.basename(os.readlink(manifest_sym_path)) == 'minilayout.xml':
_AssertNotMiniLayout()
# For now, we only support the add command.
assert options.command == 'add'
if options.workon:
if options.path is not None:
parser.error('Adding workon projects do not set project.')
else:
if options.remote is None:
parser.error('Adding non-workon projects requires a remote.')
if options.path is None:
parser.error('Adding non-workon projects requires a path.')
name = options.project
path = options.path
revision = options.revision
if revision is not None:
if (not git.IsRefsTags(revision) and not git.IsSHA1(revision)):
revision = git.StripRefsHeads(revision, False)
main_manifest = git.ManifestCheckout(os.getcwd())
main_element = main_manifest.FindCheckouts(name)
if path is not None:
main_element_from_path = main_manifest.FindCheckoutFromPath(
path, strict=False)
if main_element_from_path is not None:
main_element.append(main_element_from_path)
local_manifest = LocalManifest.FromPath(options.local_manifest_path)
if options.workon:
if not main_element:
parser.error('No project named %r in the default manifest.' % name)
_AddProjectsToManifestGroups(
options, [checkout['name'] for checkout in main_element])
elif main_element:
if options.remote is not None:
# Likely this project wasn't meant to be remote, so workon main element
print(
"Project already exists in manifest. Using that as workon project."
)
_AddProjectsToManifestGroups(
options, [checkout['name'] for checkout in main_element])
else:
# Conflict will occur; complain.
parser.error(
"Requested project name=%r path=%r will conflict with "
"your current manifest %s" %
(name, path, main_manifest.manifest_path))
elif local_manifest.GetProject(name, path=path) is not None:
parser.error("Requested project name=%r path=%r conflicts with "
"your local_manifest.xml" % (name, path))
else:
element = local_manifest.AddNonWorkonProject(name=name,
path=path,
remote=options.remote,
revision=revision)
_AddProjectsToManifestGroups(options, [element.attrib['name']])
with open(options.local_manifest_path, 'w') as f:
f.write(local_manifest.ToString())
return 0
def _ProcessCheckout(self, src_manifest, src_checkout):
"""Performs per-checkout push operations.
Args:
src_manifest: The ManifestCheckout object for the current manifest.
src_checkout: The ProjectCheckout object to process.
"""
if not src_checkout.IsBranchableProject():
# We don't have the ability to push branches to this repository. Just
# use TOT instead.
return
checkout_name = src_checkout['name']
remote = src_checkout['push_remote']
src_ref = src_checkout['revision']
suffix = self._GetBranchSuffix(src_manifest, src_checkout)
# The source/destination branches depend on options.
if self.rename_to:
# Rename flow. Both src and dst branches exist.
src_branch = '%s%s' % (self.branch_name, suffix)
dst_branch = '%s%s' % (self.rename_to, suffix)
elif self._run.options.delete_branch:
# Delete flow. Only dst branch exists.
src_branch = None
dst_branch = '%s%s' % (self.branch_name, suffix)
else:
# Create flow (default). Only dst branch exists. Source
# for the branch will just be src_ref.
src_branch = None
dst_branch = '%s%s' % (self.branch_name, suffix)
# Normalize branch refs to remote. We only process remote branches.
src_branch = git.NormalizeRemoteRef(remote, src_branch)
dst_branch = git.NormalizeRemoteRef(remote, dst_branch)
# Determine whether src/dst branches exist now, by getting their sha1s.
if src_branch:
src_sha1 = self._GetSHA1(src_checkout, src_branch)
elif git.IsSHA1(src_ref):
src_sha1 = src_ref
dst_sha1 = self._GetSHA1(src_checkout, dst_branch)
# Complain if the branch already exists, unless that is expected.
force = self._run.options.force_create or self._run.options.delete_branch
if dst_sha1 and not force:
# We are either creating a branch or renaming a branch, and the
# destination branch unexpectedly exists. Accept this only if the
# destination branch is already at the revision we want.
if src_sha1 != dst_sha1:
raise BranchError(
'Checkout %s already contains branch %s. Run with '
'--force-create to overwrite.' %
(checkout_name, dst_branch))
logging.info(
'Checkout %s already contains branch %s and it already'
' points to revision %s', checkout_name, dst_branch, dst_sha1)
elif self._run.options.delete_branch:
# Delete the dst_branch, if it exists.
if dst_sha1:
self._DeleteBranch(src_checkout, dst_branch)
else:
raise BranchError(
'Checkout %s does not contain branch %s to delete.' %
(checkout_name, dst_branch))
elif self.rename_to:
# Copy src_branch to dst_branch, if it exists, then delete src_branch.
if src_sha1:
self._CopyBranch(src_checkout, src_branch, dst_branch)
self._DeleteBranch(src_checkout, src_branch)
else:
raise BranchError(
'Checkout %s does not contain branch %s to rename.' %
(checkout_name, src_branch))
else:
# Copy src_ref to dst_branch.
self._CopyBranch(src_checkout,
src_ref,
dst_branch,
force=self._run.options.force_create)
def main(argv):
parser = optparse.OptionParser(usage='usage: %prog add [options] <name> '
'<--workon | <path> --remote <remote> >')
parser.add_option('-w',
'--workon',
action='store_true',
dest='workon',
default=False,
help='Is this a workon package?')
parser.add_option('-r', '--remote', dest='remote', default=None)
parser.add_option('-v',
'--revision',
dest='revision',
default=None,
help="Use to override the manifest defined default "
"revision used for a given project.")
parser.add_option('--upgrade-minilayout',
default=False,
action='store_true',
help="Upgrade a minilayout checkout into a full.xml "
"checkout utilizing manifest groups.")
(options, args) = parser.parse_args(argv)
repo_dir = git.FindRepoDir(os.getcwd())
if not repo_dir:
parser.error("This script must be invoked from within a repository "
"checkout.")
options.git_config = os.path.join(repo_dir, 'manifests.git', 'config')
options.repo_dir = repo_dir
options.local_manifest_path = os.path.join(repo_dir, 'local_manifest.xml')
# This constant is used only when we're doing an upgrade away from
# minilayout.xml to default.xml.
options.default_manifest_path = os.path.join(repo_dir, 'manifests',
'default.xml')
options.manifest_sym_path = os.path.join(repo_dir, 'manifest.xml')
active_manifest = os.path.basename(os.readlink(options.manifest_sym_path))
upgrade_required = active_manifest == 'minilayout.xml'
if options.upgrade_minilayout:
if args:
parser.error("--upgrade-minilayout takes no arguments.")
if not upgrade_required:
print "This repository checkout isn't using minilayout.xml; nothing to do"
else:
_UpgradeMinilayout(options)
return 0
elif upgrade_required:
logging.warn(
"Your repository checkout is using the old minilayout.xml workflow; "
"auto-upgrading it.")
cros_build_lib.RunCommand([sys.argv[0], '--upgrade-minilayout'],
cwd=os.getcwd(),
print_cmd=False)
if not args:
parser.error("No command specified.")
elif args[0] != 'add':
parser.error("Only supported subcommand is add right now.")
elif options.workon:
if len(args) != 2:
parser.error(
"Argument count is wrong for --workon; must be add <project>")
name, path = args[1], None
else:
if options.remote is None:
parser.error('Adding non-workon projects requires a remote.')
elif len(args) != 3:
parser.error("Argument count is wrong for non-workon mode; "
"must be add <project> <path> --remote <remote-arg>")
name, path = args[1:]
revision = options.revision
if revision is not None:
if (not git.IsRefsTags(revision) and not git.IsSHA1(revision)):
revision = git.StripRefsHeads(revision, False)
main_manifest = Manifest.FromPath(options.manifest_sym_path,
empty_if_missing=False)
local_manifest = Manifest.FromPath(options.local_manifest_path)
main_element = main_manifest.GetProject(name, path=path)
if options.workon:
if main_element is None:
parser.error('No project named %r in the default manifest.' % name)
_AddProjectsToManifestGroups(options, main_element.attrib['name'])
elif main_element is not None:
if options.remote is not None:
# Likely this project wasn't meant to be remote, so workon main element
print "Project already exists in manifest. Using that as workon project."
_AddProjectsToManifestGroups(options, main_element.attrib['name'])
else:
# Conflict will occur; complain.
parser.error(
"Requested project name=%r path=%r will conflict with "
"your current manifest %s" % (name, path, active_manifest))
elif local_manifest.GetProject(name, path=path) is not None:
parser.error("Requested project name=%r path=%r conflicts with "
"your local_manifest.xml" % (name, path))
else:
element = local_manifest.AddNonWorkonProject(name=name,
path=path,
remote=options.remote,
revision=revision)
_AddProjectsToManifestGroups(options, element.attrib['name'])
with open(options.local_manifest_path, 'w') as f:
f.write(local_manifest.ToString())
return 0
def main(argv):
parser = GetParser()
options = parser.parse_args(argv)
repo_dir = git.FindRepoDir(os.getcwd())
if not repo_dir:
parser.error("This script must be invoked from within a repository "
"checkout.")
options.git_config = os.path.join(repo_dir, 'manifests.git', 'config')
options.repo_dir = repo_dir
options.local_manifest_path = os.path.join(repo_dir, 'local_manifest.xml')
# This constant is used only when we're doing an upgrade away from
# minilayout.xml to default.xml.
options.default_manifest_path = os.path.join(repo_dir, 'manifests',
'default.xml')
options.manifest_sym_path = os.path.join(repo_dir, 'manifest.xml')
active_manifest = os.path.basename(os.readlink(options.manifest_sym_path))
upgrade_required = active_manifest == 'minilayout.xml'
if options.command == 'upgrade-minilayout':
if not upgrade_required:
print("This repository checkout isn't using minilayout.xml; "
"nothing to do")
else:
_UpgradeMinilayout(options)
return 0
elif upgrade_required:
logging.warning(
"Your repository checkout is using the old minilayout.xml workflow; "
"auto-upgrading it.")
main(['upgrade-minilayout'])
# For now, we only support the add command.
assert options.command == 'add'
if options.workon:
if options.path is not None:
parser.error('Adding workon projects do not set project.')
else:
if options.remote is None:
parser.error('Adding non-workon projects requires a remote.')
if options.path is None:
parser.error('Adding non-workon projects requires a path.')
name = options.project
path = options.path
revision = options.revision
if revision is not None:
if (not git.IsRefsTags(revision) and not git.IsSHA1(revision)):
revision = git.StripRefsHeads(revision, False)
main_manifest = Manifest.FromPath(options.manifest_sym_path,
empty_if_missing=False)
local_manifest = Manifest.FromPath(options.local_manifest_path)
main_element = main_manifest.GetProject(name, path=path)
if options.workon:
if main_element is None:
parser.error('No project named %r in the default manifest.' % name)
_AddProjectsToManifestGroups(options, main_element.attrib['name'])
elif main_element is not None:
if options.remote is not None:
# Likely this project wasn't meant to be remote, so workon main element
print(
"Project already exists in manifest. Using that as workon project."
)
_AddProjectsToManifestGroups(options, main_element.attrib['name'])
else:
# Conflict will occur; complain.
parser.error(
"Requested project name=%r path=%r will conflict with "
"your current manifest %s" % (name, path, active_manifest))
elif local_manifest.GetProject(name, path=path) is not None:
parser.error("Requested project name=%r path=%r conflicts with "
"your local_manifest.xml" % (name, path))
else:
element = local_manifest.AddNonWorkonProject(name=name,
path=path,
remote=options.remote,
revision=revision)
_AddProjectsToManifestGroups(options, element.attrib['name'])
with open(options.local_manifest_path, 'w') as f:
f.write(local_manifest.ToString())
return 0
