def WasStageSuccessful(self, name):
"""Return true if stage passed."""
cros_build_lib.Info('Checking for %s' % name)
for entry in self._results_log:
entry, result, _, _ = entry
if entry == name:
cros_build_lib.Info('Found %s' % result)
return result == self.SUCCESS
return False
def CleanStalePackages(boards, package_atoms):
"""Cleans up stale package info from a previous build.
Args:
boards: Boards to clean the packages from.
package_atoms: The actual package atom to unmerge.
"""
if package_atoms:
cros_build_lib.Info('Cleaning up stale packages %s.' % package_atoms)
# First unmerge all the packages for a board, then eclean it.
# We need these two steps to run in order (unmerge/eclean),
# but we can let all the boards run in parallel.
def _CleanStalePackages(board):
if board:
suffix = '-' + board
runcmd = cros_build_lib.RunCommand
else:
suffix = ''
runcmd = cros_build_lib.SudoRunCommand
if package_atoms:
runcmd(['emerge' + suffix, '-q', '--unmerge'] + package_atoms,
extra_env={'CLEAN_DELAY': '0'})
runcmd(['eclean' + suffix, '-d', 'packages'],
redirect_stdout=True, redirect_stderr=True)
tasks = []
for board in boards:
tasks.append([board])
tasks.append([None])
parallel.RunTasksInProcessPool(_CleanStalePackages, tasks)
def _GetSpecificVersionUrl(base_url, revision, time_to_wait=600):
"""Returns the Chromium version, from a repository URL and version.
Args:
base_url: URL for the root of the chromium checkout.
revision: the SVN revision we want to use.
time_to_wait: the minimum period before abandoning our wait for the
desired revision to be present.
"""
svn_url = os.path.join(_GetSvnUrl(base_url), 'src', 'chrome', 'VERSION')
if not revision or not (int(revision) > 0):
raise Exception('Revision must be positive, got %s' % revision)
start = time.time()
# Use the fact we are SVN, hence ordered.
# Dodge the fact it will silently ignore the revision if it is not
# yet known. (i.e. too high)
repo_version = gclient.GetTipOfTrunkSvnRevision(base_url)
while revision > repo_version:
if time.time() - start > time_to_wait:
raise Exception('Timeout Exceeeded')
msg = 'Repository only has version %s, looking for %s. Sleeping...'
cros_build_lib.Info(msg, repo_version, revision)
time.sleep(30)
repo_version = gclient.GetTipOfTrunkSvnRevision(base_url)
chrome_version_info = cros_build_lib.RunCommand(
['svn', 'cat', '-r', revision, svn_url],
redirect_stdout=True,
error_message='Could not read version file at %s revision %s.' %
(svn_url, revision)).output
return _GetVersionContents(chrome_version_info)
def _TreeStatusTestHelper(self, tree_status, general_state, expected_return,
retries_500=0, max_timeout=0):
"""Tests whether we return the correct value based on tree_status."""
return_status = self._TreeStatusFile(tree_status, general_state)
self.mox.StubOutWithMock(urllib, 'urlopen')
status_url = 'https://chromiumos-status.appspot.com/current?format=json'
backoff = 1
sleep_timeout = 1
for _attempt in range(retries_500):
urllib.urlopen(status_url).AndReturn(return_status)
return_status.getcode().AndReturn(500)
time.sleep(backoff)
backoff *= 2
urllib.urlopen(status_url).MultipleTimes().AndReturn(return_status)
# Time is checked twice to bootstrap.
start_time = 1
self.mox.StubOutWithMock(time, 'time')
time.time().AndReturn(start_time)
time.time().AndReturn(start_time)
if expected_return == False:
for time_plus in xrange(max_timeout + 1):
time.time().AndReturn(start_time + time_plus)
self.mox.StubOutWithMock(cros_build_lib, 'Info')
cros_build_lib.Info(mox.IgnoreArg(), mox.IgnoreArg()).MultipleTimes()
time.sleep(sleep_timeout).MultipleTimes()
return_status.getcode().MultipleTimes().AndReturn(200)
return_status.read().MultipleTimes().AndReturn(return_status.json)
self.mox.ReplayAll()
self.assertEqual(cros_build_lib.TreeOpen(status_url, sleep_timeout,
max_timeout), expected_return)
self.mox.VerifyAll()
def Log(msg):
"""Print out a message if we are in verbose mode.
Args:
msg: Message to print
"""
if verbose:
cros_build_lib.Info(msg)
def PushChange(stable_branch, tracking_branch, dryrun, cwd):
"""Pushes commits in the stable_branch to the remote git repository.
Pushes local commits from calls to CommitChange to the remote git
repository specified by current working directory. If changes are
found to commit, they will be merged to the merge branch and pushed.
In that case, the local repository will be left on the merge branch.
Args:
stable_branch: The local branch with commits we want to push.
tracking_branch: The tracking branch of the local branch.
dryrun: Use git push --dryrun to emulate a push.
cwd: The directory to run commands in.
Raises:
OSError: Error occurred while pushing.
"""
if not _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd):
cros_build_lib.Info('No work found to push in %s. Exiting', cwd)
return
# For the commit queue, our local branch may contain commits that were
# just tested and pushed during the CommitQueueCompletion stage. Sync
# and rebase our local branch on top of the remote commits.
remote, push_branch = git.GetTrackingBranch(cwd, for_push=True)
git.SyncPushBranch(cwd, remote, push_branch)
# Check whether any local changes remain after the sync.
if not _DoWeHaveLocalCommits(stable_branch, push_branch, cwd):
cros_build_lib.Info('All changes already pushed for %s. Exiting', cwd)
return
description = cros_build_lib.RunCommandCaptureOutput([
'git', 'log', '--format=format:%s%n%n%b',
'%s..%s' % (push_branch, stable_branch)
],
cwd=cwd).output
description = 'Marking set of ebuilds as stable\n\n%s' % description
cros_build_lib.Info('For %s, using description %s', cwd, description)
git.CreatePushBranch(constants.MERGE_BRANCH, cwd)
git.RunGit(cwd, ['merge', '--squash', stable_branch])
git.RunGit(cwd, ['commit', '-m', description])
git.RunGit(cwd, ['config', 'push.default', 'tracking'])
git.PushWithRetry(constants.MERGE_BRANCH, cwd, dryrun=dryrun)
def GenerateBlameList(source_repo, lkgm_path, only_print_chumps=False):
"""Generate the blamelist since the specified manifest.
Arguments:
source_repo: Repository object for the source code.
lkgm_path: Path to LKGM manifest.
only_print_chumps: If True, only print changes that were chumped.
"""
handler = git.Manifest(lkgm_path)
reviewed_on_re = re.compile(r'\s*Reviewed-on:\s*(\S+)')
author_re = re.compile(r'\s*Author:.*<(\S+)@\S+>\s*')
committer_re = re.compile(r'\s*Commit:.*<(\S+)@\S+>\s*')
for project in handler.projects.keys():
rel_src_path = handler.projects[project].get('path')
# If it's not part of our source tree, it doesn't affect our build.
if not rel_src_path:
continue
# Additional case in case the repo has been removed from the manifest.
src_path = source_repo.GetRelativePath(rel_src_path)
if not os.path.exists(src_path):
cros_build_lib.Info('Detected repo removed from manifest %s' % project)
continue
revision = handler.projects[project]['revision']
result = cros_build_lib.RunCommand(['git', 'log', '--pretty=full',
'%s..HEAD' % revision],
print_cmd=False, redirect_stdout=True,
cwd=src_path)
current_author = None
current_committer = None
for line in unicode(result.output, 'ascii', 'ignore').splitlines():
author_match = author_re.match(line)
if author_match:
current_author = author_match.group(1)
committer_match = committer_re.match(line)
if committer_match:
current_committer = committer_match.group(1)
review_match = reviewed_on_re.match(line)
if review_match:
review = review_match.group(1)
_, _, change_number = review.rpartition('/')
if current_committer != 'chrome-bot':
cros_build_lib.PrintBuildbotLink(
'CHUMP %s:%s' % (current_author, change_number),
review)
elif not only_print_chumps:
cros_build_lib.PrintBuildbotLink(
'%s:%s' % (current_author, change_number),
review)
def DownloadCrx(ext, extension, outputdir):
"""Download .crx file from WebStore and update entry."""
cros_build_lib.Info('Extension "%s"(%s)...', extension['name'], ext)
update_url = '%s?x=id%%3D%s%%26uc' % (extension['external_update_url'],
ext)
response = urllib.urlopen(update_url)
if response.getcode() != 200:
cros_build_lib.Error('Cannot get update response, URL: %s, error: %d',
update_url, response.getcode())
return False
dom = xml.dom.minidom.parse(response)
status = dom.getElementsByTagName('app')[0].getAttribute('status')
if status != 'ok':
cros_build_lib.Error('Cannot fetch extension, status: %s', status)
return False
node = dom.getElementsByTagName('updatecheck')[0]
url = node.getAttribute('codebase')
version = node.getAttribute('version')
filename = '%s-%s.crx' % (ext, version)
response = urllib.urlopen(url)
if response.getcode() != 200:
cros_build_lib.Error('Cannot download extension, URL: %s, error: %d',
url, response.getcode())
return False
osutils.WriteFile(os.path.join(outputdir, 'extensions', filename),
response.read())
# Has to delete because only one of 'external_crx' or
# 'external_update_url' should present for the extension.
del extension['external_update_url']
extension['external_crx'] = os.path.join(EXTENSIONS_CACHE_PREFIX, filename)
extension['external_version'] = version
cros_build_lib.Info('Downloaded, current version %s', version)
return True
def GetNextVersion(self, version_info):
"""Returns the next version string that should be built."""
version = version_info.VersionString()
if self.latest == version:
message = (
'Automatic: %s - Updating to a new version number from %s' %
(self.build_name, version))
version = version_info.IncrementVersion(message,
dry_run=self.dry_run)
assert version != self.latest
cros_build_lib.Info('Incremented version number to %s', version)
return version
class _Lock(cros_build_lib.MasterPidContextManager):
"""Base lockf based locking. Derivatives need to override _GetFd"""
def __init__(self, description=None, verbose=True):
"""Initialize this instance.
Args:
path: On disk pathway to lock. Can be a directory or a file.
description: A description for this lock- what is it protecting?
"""
cros_build_lib.MasterPidContextManager.__init__(self)
self._verbose = verbose
self.description = description
self._fd = None
@property
def fd(self):
if self._fd is None:
self._fd = self._GetFd()
# Ensure that all derivatives of this lock can't bleed the fd
# across execs.
fcntl.fcntl(self._fd, fcntl.F_SETFD,
fcntl.fcntl(self._fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
return self._fd
def _GetFd(self):
raise NotImplementedError(self, '_GetFd')
def _enforce_lock(self, flags, message):
# Try nonblocking first, if it fails, display the context/message,
# and then wait on the lock.
try:
fcntl.lockf(self.fd, flags|fcntl.LOCK_NB)
return
except EnvironmentError, e:
if e.errno == errno.EDEADLOCK:
self.unlock()
elif e.errno != errno.EAGAIN:
raise
if self.description:
message = '%s: blocking while %s' % (self.description, message)
if self._verbose:
cros_build_lib.Info(message)
try:
fcntl.lockf(self.fd, flags)
except EnvironmentError, e:
if e.errno != errno.EDEADLOCK:
raise
self.unlock()
fcntl.lockf(self.fd, flags)
def GetTipOfTrunkSvnRevision(svn_url):
"""Returns the current svn revision for the chrome tree."""
cmd = ['svn', 'info', svn_url]
svn_info = cros_build_lib.RunCommand(cmd, redirect_stdout=True).output
revision_re = re.compile('^Revision:\s+(\d+)')
for line in svn_info.splitlines():
match = revision_re.match(line)
if match:
svn_revision = match.group(1)
cros_build_lib.Info('Found SVN Revision %s' % svn_revision)
return svn_revision
raise Exception('Could not find revision information from %s' % svn_url)
def PushWithRetry(branch, git_repo, dryrun=False, retries=5):
"""General method to push local git changes.
This method only works with branches created via the CreatePushBranch
function.
Args:
branch: Local branch to push. Branch should have already been created
with a local change committed ready to push to the remote branch. Must
also already be checked out to that branch.
git_repo: Git repository to push from.
dryrun: Git push --dry-run if set to True.
retries: The number of times to retry before giving up, default: 5
Raises:
GitPushFailed if push was unsuccessful after retries
"""
remote, ref = GetTrackingBranch(git_repo,
branch,
for_checkout=False,
for_push=True)
# Don't like invoking this twice, but there is a bit of API
# impedence here; cros_mark_as_stable
_, local_ref = GetTrackingBranch(git_repo, branch, for_push=True)
if not ref.startswith("refs/heads/"):
raise Exception("Was asked to push to a non branch namespace: %s" %
(ref, ))
push_command = ['push', remote, '%s:%s' % (branch, ref)]
cros_build_lib.Debug("Trying to push %s to %s:%s", git_repo, branch, ref)
if dryrun:
push_command.append('--dry-run')
for retry in range(1, retries + 1):
SyncPushBranch(git_repo, remote, local_ref)
try:
RunGit(git_repo, push_command)
break
except cros_build_lib.RunCommandError:
if retry < retries:
Warning('Error pushing changes trying again (%s/%s)', retry,
retries)
time.sleep(5 * retry)
continue
raise
cros_build_lib.Info("Successfully pushed %s to %s:%s", git_repo, branch,
ref)
def main(argv):
parser = commandline.ArgumentParser('%%(prog)s [options] <version>\n\n%s' %
__doc__)
parser.add_argument('version', nargs=1)
parser.add_argument('--path',
default=None,
type='path',
help='Path of files dir with external_extensions.json')
parser.add_argument('--create',
default=False,
action='store_true',
help='Create cache tarball with specified name')
parser.add_argument('--upload',
default=False,
action='store_true',
help='Upload cache tarball with specified name')
options = parser.parse_args(argv)
if options.path:
os.chdir(options.path)
if not (options.create or options.upload):
cros_build_lib.Die('Need at least --create or --upload args')
if not os.path.exists('external_extensions.json'):
cros_build_lib.Die(
'No external_extensions.json in %s. Did you forget the '
'--path option?', os.getcwd())
tarball = '%s.tar.xz' % options.version[0]
if options.create:
extensions = json.load(open('external_extensions.json', 'r'))
with osutils.TempDir() as tempdir:
CreateCacheTarball(extensions, tempdir, os.path.abspath(tarball))
if options.upload:
ctx = gs.GSContext()
url = os.path.join(UPLOAD_URL_BASE, tarball)
if ctx.Exists(url):
cros_build_lib.Die(
'This version already exists on Google Storage (%s)!\n'
'NEVER REWRITE EXISTING FILE. IT WILL BREAK CHROME OS '
'BUILD!!!', url)
ctx.Copy(os.path.abspath(tarball), url, acl='project-private')
cros_build_lib.Info('Tarball uploaded %s', url)
osutils.SafeUnlink(os.path.abspath(tarball))
def CreateCacheTarball(extensions, outputdir, tarball):
"""Cache |extensions| in |outputdir| and pack them in |tarball|."""
osutils.SafeMakedirs(os.path.join(outputdir, 'extensions',
'managed_users'))
was_errors = False
for ext in extensions:
managed_users = extensions[ext].get('managed_users', 'no')
cache_crx = extensions[ext].get('cache_crx', 'yes')
# Remove fields that shouldn't be in the output file.
for key in ('cache_crx', 'managed_users'):
extensions[ext].pop(key, None)
if cache_crx == 'yes':
if not DownloadCrx(ext, extensions[ext], outputdir):
was_errors = True
elif cache_crx == 'no':
pass
else:
cros_build_lib.Die('Unknown value for "cache_crx" %s for %s',
cache_crx, ext)
if managed_users == 'yes':
json_file = os.path.join(outputdir,
'extensions/managed_users/%s.json' % ext)
json.dump(extensions[ext],
open(json_file, 'w'),
sort_keys=True,
indent=2,
separators=(',', ': '))
if managed_users != 'only':
json_file = os.path.join(outputdir, 'extensions/%s.json' % ext)
json.dump(extensions[ext],
open(json_file, 'w'),
sort_keys=True,
indent=2,
separators=(',', ': '))
if was_errors:
cros_build_lib.Die('FAIL to download some extensions')
cros_build_lib.CreateTarball(tarball, outputdir)
cros_build_lib.Info('Tarball created %s', tarball)
def _GetFilesForTarget(target, root='/'):
"""Locate all the files to package for |target|
This does not cover ELF dependencies.
Args:
target: The toolchain target name
root: The root path to pull all packages from
Returns:
A tuple of a set of all packable paths, and a set of all paths which
are also native ELFs
"""
paths = set()
elfs = set()
# Find all the files owned by the packages for this target.
for pkg in GetTargetPackages(target):
# Ignore packages that are part of the target sysroot.
if pkg in ('kernel', 'libc'):
continue
atom = GetPortagePackage(target, pkg)
cat, pn = atom.split('/')
ver = GetInstalledPackageVersions(atom)[0]
cros_build_lib.Info('packaging %s-%s', atom, ver)
# pylint: disable=E1101
dblink = portage.dblink(cat,
pn + '-' + ver,
myroot=root,
settings=portage.settings)
contents = dblink.getcontents()
for obj in contents:
ptype = contents[obj][0]
if not IsPathPackagable(ptype, obj):
continue
if ptype == 'obj':
# For native ELFs, we need to pull in their dependencies too.
if FileIsCrosSdkElf(obj):
elfs.add(obj)
paths.add(obj)
return paths, elfs
def __init__(self, options, bots, local_patches):
"""Construct the object.
Args:
options: The parsed options passed into cbuildbot.
bots: A list of configs to run tryjobs for.
local_patches: A list of LocalPatch objects.
"""
self.options = options
self.user = getpass.getuser()
cwd = os.path.dirname(os.path.realpath(__file__))
self.user_email = git.GetProjectUserEmail(cwd)
cros_build_lib.Info('Using email:%s', self.user_email)
# Name of the job that appears on the waterfall.
patch_list = options.gerrit_patches + options.local_patches
self.name = options.remote_description
if self.name is None:
self.name = ''
if options.branch != 'master':
self.name = '[%s] ' % options.branch
self.name += ','.join(patch_list)
self.bots = bots[:]
self.slaves_request = options.slaves
self.description = ('name: %s\n patches: %s\nbots: %s' %
(self.name, patch_list, self.bots))
self.extra_args = options.pass_through_args
if '--buildbot' not in self.extra_args:
self.extra_args.append('--remote-trybot')
self.extra_args.append('--remote-version=%s'
% (self.TRYJOB_FORMAT_VERSION,))
self.tryjob_repo = None
self.local_patches = local_patches
self.ssh_url = self.EXT_SSH_URL
self.manifest = None
if repository.IsARepoRoot(options.sourceroot):
self.manifest = git.ManifestCheckout.Cached(options.sourceroot)
if repository.IsInternalRepoCheckout(options.sourceroot):
self.ssh_url = self.INT_SSH_URL
def CleanStalePackages(boards, package_atoms):
"""Cleans up stale package info from a previous build.
Args:
boards: Boards to clean the packages from.
package_atoms: A list of package atoms to unmerge.
"""
if package_atoms:
cros_build_lib.Info('Cleaning up stale packages %s.' % package_atoms)
# First unmerge all the packages for a board, then eclean it.
# We need these two steps to run in order (unmerge/eclean),
# but we can let all the boards run in parallel.
def _CleanStalePackages(board):
if board:
suffix = '-' + board
runcmd = cros_build_lib.RunCommand
else:
suffix = ''
runcmd = cros_build_lib.SudoRunCommand
emerge, eclean = 'emerge' + suffix, 'eclean' + suffix
if not osutils.FindMissingBinaries([emerge, eclean]):
# If nothing was found to be unmerged, emerge will exit(1).
result = runcmd([emerge, '-q', '--unmerge'] + package_atoms,
extra_env={'CLEAN_DELAY': '0'},
error_code_ok=True)
if not result.returncode in (0, 1):
raise cros_build_lib.RunCommandError('unexpected error',
result)
runcmd([eclean, '-d', 'packages'],
redirect_stdout=True,
redirect_stderr=True)
tasks = []
for board in boards:
tasks.append([board])
tasks.append([None])
parallel.RunTasksInProcessPool(_CleanStalePackages, tasks)
def Run(self):
cros_build_lib.Info('My options are %r', self.options)
def WriteFirmware(options):
"""Write firmware to the board.
This uses cros_bundle_firmware to create a firmware image and write it to
the board.
Args:
options: Command line options
"""
flash = []
kernel = []
run = []
secure = []
servo = []
silent = []
verbose_arg = []
ro_uboot = []
bl2 = ['--bl2', '%s/spl/%s-spl.bin' % (outdir, smdk)]
if options.use_defaults:
bl1 = []
bmpblk = []
ecro = []
ecrw = []
defaults = []
else:
bl1 = ['--bl1', '##/build/%s/firmware/u-boot.bl1.bin' % options.board]
bmpblk = [
'--bmpblk',
'##/build/%s/firmware/bmpblk.bin' % options.board
]
ecro = ['--ecro', '##/build/%s/firmware/ec.RO.bin' % options.board]
ecrw = ['--ec', '##/build/%s/firmware/ec.RW.bin' % options.board]
defaults = ['-D']
if arch == 'x86':
seabios = [
'--seabios',
'##/build/%s/firmware/seabios.cbfs' % options.board
]
else:
seabios = []
if options.sdcard:
dest = 'sd:.'
elif arch == 'x86':
dest = 'em100'
elif arch == 'sandbox':
dest = ''
else:
dest = 'usb'
port = SERVO_PORT.get(options.board, '')
if port:
servo = ['--servo', '%d' % port]
if options.flash:
flash = ['-F', 'spi']
# The small builds don't have the command line interpreter so cannot
# run the magic flasher script. So use the standard U-Boot in this
# case.
if options.small:
cros_build_lib.Warning('Using standard U-Boot as flasher')
flash += ['-U', '##/build/%s/firmware/u-boot.bin' % options.board]
if options.mmc:
flash = ['-F', 'sdmmc']
if options.verbose:
verbose_arg = ['-v', '%s' % options.verbose]
if options.secure:
secure += ['--bootsecure', '--bootcmd', 'vboot_twostop']
if not options.verified:
# Make a small image, without GBB, etc.
secure.append('-s')
if options.kernel:
kernel = ['--kernel', '##/build/%s/boot/vmlinux.uimg' % options.board]
if not options.console:
silent = ['--add-config-int', 'silent-console', '1']
if not options.run:
run = ['--bootcmd', 'none']
if arch != 'sandbox' and not in_chroot and servo:
if dest == 'usb':
cros_build_lib.Warning('Image cannot be written to board')
dest = ''
servo = []
elif dest == 'em100':
cros_build_lib.Warning(
'Please reset the board manually to boot firmware')
servo = []
if not servo:
cros_build_lib.Warning(
'(sadly dut-control does not work outside chroot)')
if dest:
dest = ['-w', dest]
else:
dest = []
soc = SOCS.get(board)
if not soc:
soc = SOCS.get(uboard, '')
dt_name = DEFAULT_DTS.get(options.board, options.board)
dts_file = 'board/%s/dts/%s%s.dts' % (vendor, soc, dt_name)
Log('Device tree: %s' % dts_file)
if arch == 'sandbox':
uboot_fname = '%s/u-boot' % outdir
else:
uboot_fname = '%s/u-boot.bin' % outdir
if options.ro:
# RO U-Boot is passed through as blob 'ro-boot'. We use the standard
# ebuild one as RW.
# TODO([email protected]): Option to build U-Boot a second time to get
# a fresh RW U-Boot.
cros_build_lib.Warning('Using standard U-Boot for RW')
ro_uboot = ['--add-blob', 'ro-boot', uboot_fname]
uboot_fname = '##/build/%s/firmware/u-boot.bin' % options.board
cbf = [
'%s/platform/dev/host/cros_bundle_firmware' % src_root, '-b',
options.board, '-d', dts_file, '-I',
'arch/%s/dts' % arch, '-I', 'cros/dts', '-u', uboot_fname, '-O',
'%s/out' % outdir, '-M', family
]
for other in [
bl1, bl2, bmpblk, defaults, dest, ecro, ecrw, flash, kernel, run,
seabios, secure, servo, silent, verbose_arg, ro_uboot
]:
if other:
cbf += other
if options.cbfargs:
for item in options.cbfargs:
cbf += item.split(' ')
os.environ['PYTHONPATH'] = ('%s/platform/dev/host/lib:%s/..' %
(src_root, src_root))
Log(' '.join(cbf))
result = cros_build_lib.RunCommand(cbf, **kwargs)
if result.returncode:
cros_build_lib.Die('cros_bundle_firmware failed')
if not dest or not result.returncode:
cros_build_lib.Info('Image is available at %s/out/image.bin' % outdir)
else:
if result.returncode:
cros_build_lib.Die('Failed to write image to board')
else:
cros_build_lib.Info('Image written to board with %s' %
' '.join(dest + servo))
def main(_argv):
usage_options = '|'.join(constants.VALID_CHROME_REVISIONS)
usage = '%s OPTIONS [%s]' % (__file__, usage_options)
parser = optparse.OptionParser(usage)
parser.add_option('-b', '--boards', default='x86-generic')
parser.add_option('-c', '--chrome_url', default=gclient.GetBaseURLs()[0])
parser.add_option('-f', '--force_revision', default=None)
parser.add_option('-s',
'--srcroot',
default=os.path.join(os.environ['HOME'], 'trunk', 'src'),
help='Path to the src directory')
parser.add_option('-t',
'--tracking_branch',
default='cros/master',
help='Branch we are tracking changes against')
(options, args) = parser.parse_args()
if len(args) != 1 or args[0] not in constants.VALID_CHROME_REVISIONS:
parser.error('Commit requires arg set to one of %s.' %
constants.VALID_CHROME_REVISIONS)
overlay_dir = os.path.abspath(_CHROME_OVERLAY_DIR %
{'srcroot': options.srcroot})
chrome_rev = args[0]
version_to_uprev = None
commit_to_use = None
sticky_branch = None
(unstable_ebuild, stable_ebuilds) = FindChromeCandidates(overlay_dir)
if chrome_rev == constants.CHROME_REV_LOCAL:
if 'CHROME_ROOT' in os.environ:
chrome_root = os.environ['CHROME_ROOT']
else:
chrome_root = os.path.join(os.environ['HOME'], 'chrome_root')
version_to_uprev = _GetTipOfTrunkVersionFile(chrome_root)
commit_to_use = 'Unknown'
cros_build_lib.Info('Using local source, versioning is untrustworthy.')
elif chrome_rev == constants.CHROME_REV_SPEC:
commit_to_use = options.force_revision
if '@' in commit_to_use:
commit_to_use = ParseMaxRevision(commit_to_use)
version_to_uprev = _GetSpecificVersionUrl(options.chrome_url,
commit_to_use)
elif chrome_rev == constants.CHROME_REV_TOT:
commit_to_use = gclient.GetTipOfTrunkSvnRevision(options.chrome_url)
version_to_uprev = _GetSpecificVersionUrl(options.chrome_url,
commit_to_use)
elif chrome_rev == constants.CHROME_REV_LATEST:
version_to_uprev = _GetLatestRelease(options.chrome_url)
else:
sticky_ebuild = _GetStickyEBuild(stable_ebuilds)
sticky_version = sticky_ebuild.chrome_version
sticky_branch = sticky_version.rpartition('.')[0]
version_to_uprev = _GetLatestRelease(options.chrome_url, sticky_branch)
stable_candidate = FindChromeUprevCandidate(stable_ebuilds, chrome_rev,
sticky_branch)
if stable_candidate:
cros_build_lib.Info('Stable candidate found %s' % stable_candidate)
else:
cros_build_lib.Info('No stable candidate found.')
tracking_branch = 'remotes/m/%s' % os.path.basename(
options.tracking_branch)
existing_branch = git.GetCurrentBranch(overlay_dir)
work_branch = cros_mark_as_stable.GitBranch(constants.STABLE_EBUILD_BRANCH,
tracking_branch, overlay_dir)
work_branch.CreateBranch()
# In the case of uprevving overlays that have patches applied to them,
# include the patched changes in the stabilizing branch.
if existing_branch:
cros_build_lib.RunCommand(['git', 'rebase', existing_branch],
cwd=overlay_dir)
chrome_version_atom = MarkChromeEBuildAsStable(stable_candidate,
unstable_ebuild, chrome_rev,
version_to_uprev,
commit_to_use, overlay_dir)
# Explicit print to communicate to caller.
if chrome_version_atom:
cros_mark_as_stable.CleanStalePackages(options.boards.split(':'),
[chrome_version_atom])
print 'CHROME_VERSION_ATOM=%s' % chrome_version_atom
def MarkChromeEBuildAsStable(stable_candidate, unstable_ebuild, chrome_rev,
chrome_version, commit, overlay_dir):
"""Uprevs the chrome ebuild specified by chrome_rev.
This is the main function that uprevs the chrome_rev from a stable candidate
to its new version.
Args:
stable_candidate: ebuild that corresponds to the stable ebuild we are
revving from. If None, builds the a new ebuild given the version
and logic for chrome_rev type with revision set to 1.
unstable_ebuild: ebuild corresponding to the unstable ebuild for chrome.
chrome_rev: one of constants.VALID_CHROME_REVISIONS or LOCAL
constants.CHROME_REV_SPEC - Requires commit value. Revs the ebuild for
the specified version and uses the portage suffix of _alpha.
constants.CHROME_REV_TOT - Requires commit value. Revs the ebuild for
the TOT version and uses the portage suffix of _alpha.
constants.CHROME_REV_LOCAL - Requires a chrome_root. Revs the ebuild for
the local version and uses the portage suffix of _alpha.
constants.CHROME_REV_LATEST - This uses the portage suffix of _rc as they
are release candidates for the next sticky version.
constants.CHROME_REV_STICKY - Revs the sticky version.
chrome_version: The \d.\d.\d.\d version of Chrome.
commit: Used with constants.CHROME_REV_TOT. The svn revision of chrome.
overlay_dir: Path to the chromeos-chrome package dir.
Returns:
Full portage version atom (including rc's, etc) that was revved.
"""
def IsTheNewEBuildRedundant(new_ebuild, stable_ebuild):
"""Returns True if the new ebuild is redundant.
This is True if there if the current stable ebuild is the exact same copy
of the new one.
"""
if not stable_ebuild:
return False
if stable_candidate.chrome_version == new_ebuild.chrome_version:
return filecmp.cmp(new_ebuild.ebuild_path,
stable_ebuild.ebuild_path,
shallow=False)
# Mark latest release and sticky branches as stable.
mark_stable = chrome_rev not in [
constants.CHROME_REV_TOT, constants.CHROME_REV_SPEC,
constants.CHROME_REV_LOCAL
]
# Case where we have the last stable candidate with same version just rev.
if stable_candidate and stable_candidate.chrome_version == chrome_version:
new_ebuild_path = '%s-r%d.ebuild' % (
stable_candidate.ebuild_path_no_revision,
stable_candidate.current_revision + 1)
else:
suffix = 'rc' if mark_stable else 'alpha'
pf = '%s-%s_%s-r1' % (constants.CHROME_PN, chrome_version, suffix)
new_ebuild_path = os.path.join(overlay_dir, '%s.ebuild' % pf)
chrome_variables = dict()
if commit:
chrome_variables[_CHROME_SVN_TAG] = commit
portage_utilities.EBuild.MarkAsStable(unstable_ebuild.ebuild_path,
new_ebuild_path,
chrome_variables,
make_stable=mark_stable)
new_ebuild = ChromeEBuild(new_ebuild_path)
# Determine whether this is ebuild is redundant.
if IsTheNewEBuildRedundant(new_ebuild, stable_candidate):
msg = 'Previous ebuild with same version found and ebuild is redundant.'
cros_build_lib.Info(msg)
os.unlink(new_ebuild_path)
return None
if stable_candidate and chrome_rev in _REV_TYPES_FOR_LINKS:
_AnnotateAndPrint(
'Chromium revisions',
GetChromeRevisionListLink(stable_candidate, new_ebuild,
chrome_rev))
cros_build_lib.RunCommand(['git', 'add', new_ebuild_path], cwd=overlay_dir)
if stable_candidate and not stable_candidate.IsSticky():
cros_build_lib.RunCommand(['git', 'rm', stable_candidate.ebuild_path],
cwd=overlay_dir)
portage_utilities.EBuild.CommitChange(
_GIT_COMMIT_MESSAGE % {
'chrome_rev': chrome_rev,
'chrome_version': chrome_version
}, overlay_dir)
return '%s-%s' % (new_ebuild.package, new_ebuild.version)
def UploadSymbol(sym_file,
upload_url,
file_limit=DEFAULT_FILE_LIMIT,
sleep=0,
num_errors=None):
"""Upload |sym_file| to |upload_url|
Args:
sym_file: The full path to the breakpad symbol to upload
upload_url: The crash server to upload things to
file_limit: The max file size of a symbol file before we try to strip it
sleep: Number of seconds to sleep before running
num_errors: An object to update with the error count (needs a .value member)
Returns:
The number of errors that were encountered.
"""
if num_errors is None:
num_errors = ctypes.c_int()
elif num_errors.value > MAX_TOTAL_ERRORS_FOR_RETRY:
# Abandon ship! It's on fire! NOoooooooooooOOOoooooo.
return 0
upload_file = sym_file
if sleep:
# Keeps us from DoS-ing the symbol server.
time.sleep(sleep)
cros_build_lib.Debug('uploading %s' % sym_file)
# Ideally there'd be a tempfile.SpooledNamedTemporaryFile that we could use.
with tempfile.NamedTemporaryFile(prefix='upload_symbols',
bufsize=0) as temp_sym_file:
if file_limit:
# If the symbols size is too big, strip out the call frame info. The CFI
# is unnecessary for 32bit x86 targets where the frame pointer is used (as
# all of ours have) and it accounts for over half the size of the symbols
# uploaded.
file_size = os.path.getsize(sym_file)
if file_size > file_limit:
cros_build_lib.Warning(
'stripping CFI from %s due to size %s > %s', sym_file,
file_size, file_limit)
temp_sym_file.writelines([
x for x in open(sym_file, 'rb').readlines()
if not x.startswith('STACK CFI')
])
upload_file = temp_sym_file.name
# Hopefully the crash server will let it through. But it probably won't.
# Not sure what the best answer is in this case.
file_size = os.path.getsize(upload_file)
if file_size > CRASH_SERVER_FILE_LIMIT:
cros_build_lib.PrintBuildbotStepWarnings()
cros_build_lib.Error(
'upload file %s is awfully large, risking rejection '
'by symbol server (%s > %s)', sym_file, file_size,
CRASH_SERVER_FILE_LIMIT)
num_errors.value += 1
# Upload the symbol file.
try:
cros_build_lib.RetryCommand(SymUpload,
MAX_RETRIES,
upload_file,
upload_url,
sleep=INITIAL_RETRY_DELAY)
cros_build_lib.Info('successfully uploaded %10i bytes: %s',
file_size, os.path.basename(sym_file))
except cros_build_lib.RunCommandError as e:
cros_build_lib.Warning(
'could not upload: %s:\n{stdout} %s\n{stderr} %s',
os.path.basename(sym_file), e.result.output, e.result.error)
num_errors.value += 1
return num_errors.value
def main(argv):
parser = commandline.ArgumentParser(description=__doc__)
parser.add_argument('sym_files', type='path', nargs='*', default=None)
parser.add_argument('--board',
default=None,
help='board to build packages for')
parser.add_argument('--breakpad_root',
type='path',
default=None,
help='root directory for breakpad symbols')
parser.add_argument('--official_build',
action='store_true',
default=False,
help='point to official symbol server')
parser.add_argument('--regenerate',
action='store_true',
default=False,
help='regenerate all symbols')
parser.add_argument('--upload-count',
type=int,
default=None,
help='only upload # number of symbols')
parser.add_argument('--strip_cfi',
type=int,
default=CRASH_SERVER_FILE_LIMIT - (10 * 1024 * 1024),
help='strip CFI data for files above this size')
parser.add_argument('--testing',
action='store_true',
default=False,
help='run in testing mode')
parser.add_argument('--yes',
action='store_true',
default=False,
help='answer yes to all prompts')
opts = parser.parse_args(argv)
if opts.sym_files:
if opts.regenerate:
cros_build_lib.Die(
'--regenerate may not be used with specific files')
else:
if opts.board is None:
cros_build_lib.Die('--board is required')
if opts.breakpad_root and opts.regenerate:
cros_build_lib.Die('--regenerate may not be used with --breakpad_root')
if opts.testing:
# TODO(build): Kill off --testing mode once unittests are up-to-snuff.
cros_build_lib.Info('running in testing mode')
# pylint: disable=W0601,W0603
global INITIAL_RETRY_DELAY, SymUpload, DEFAULT_SLEEP_DELAY
INITIAL_RETRY_DELAY = DEFAULT_SLEEP_DELAY = 0
SymUpload = TestingSymUpload
if not opts.yes:
query = textwrap.wrap(
textwrap.dedent("""
Uploading symbols for an entire Chromium OS build is really only
necessary for release builds and in a few cases for developers
to debug problems. It will take considerable time to run. For
developer debugging purposes, consider instead passing specific
files to upload.
"""), 80)
cros_build_lib.Warning('\n%s', '\n'.join(query))
if not cros_build_lib.BooleanPrompt(
prompt='Are you sure you want to upload all build symbols',
default=False):
cros_build_lib.Die('better safe than sorry')
ret = 0
if opts.regenerate:
ret += cros_generate_breakpad_symbols.GenerateBreakpadSymbols(
opts.board, breakpad_dir=opts.breakpad_root)
ret += UploadSymbols(opts.board,
official=opts.official_build,
breakpad_dir=opts.breakpad_root,
file_limit=opts.strip_cfi,
sleep=DEFAULT_SLEEP_DELAY,
upload_count=opts.upload_count,
sym_files=opts.sym_files)
if ret:
cros_build_lib.Error('encountered %i problem(s)', ret)
# Since exit(status) gets masked, clamp it to 1 so we don't inadvertently
# return 0 in case we are a multiple of the mask.
ret = 1
return ret
def UploadSymbols(board=None,
official=False,
breakpad_dir=None,
file_limit=DEFAULT_FILE_LIMIT,
sleep=DEFAULT_SLEEP_DELAY,
upload_count=None,
sym_files=None):
"""Upload all the generated symbols for |board| to the crash server
You can use in a few ways:
* pass |board| to locate all of its symbols
* pass |breakpad_dir| to upload all the symbols in there
* pass |sym_files| to upload specific symbols
Args:
board: The board whose symbols we wish to upload
official: Use the official symbol server rather than the staging one
breakpad_dir: The full path to the breakpad directory where symbols live
file_limit: The max file size of a symbol file before we try to strip it
sleep: How long to sleep in between uploads
upload_count: If set, only upload this many symbols (meant for testing)
sym_files: Specific symbol files to upload, otherwise search |breakpad_dir|
Returns:
The number of errors that were encountered.
"""
num_errors = 0
if official:
upload_url = OFFICIAL_UPLOAD_URL
else:
cros_build_lib.Warning(
'unofficial builds upload to the staging server')
upload_url = STAGING_UPLOAD_URL
if sym_files:
cros_build_lib.Info('uploading specified symbol files to %s',
upload_url)
sym_file_sets = [('', '', sym_files)]
all_files = True
else:
if breakpad_dir is None:
breakpad_dir = cros_generate_breakpad_symbols.FindBreakpadDir(
board)
cros_build_lib.Info('uploading all symbols to %s from %s', upload_url,
breakpad_dir)
sym_file_sets = os.walk(breakpad_dir)
all_files = False
# We need to limit ourselves to one upload at a time to avoid the server
# kicking in DoS protection. See these bugs for more details:
# http://crbug.com/209442
# http://crbug.com/212496
bg_errors = multiprocessing.Value('i')
with parallel.BackgroundTaskRunner(UploadSymbol,
file_limit=file_limit,
sleep=sleep,
num_errors=bg_errors,
processes=1) as queue:
for root, _, files in sym_file_sets:
if upload_count == 0:
break
for sym_file in files:
if all_files or sym_file.endswith('.sym'):
sym_file = os.path.join(root, sym_file)
queue.put([sym_file, upload_url])
if upload_count is not None:
upload_count -= 1
if upload_count == 0:
break
num_errors += bg_errors.value
return num_errors
def GenerateBreakpadSymbols(board,
breakpad_dir=None,
strip_cfi=False,
generate_count=None,
sysroot=None):
"""Generate all the symbols for this board
TODO(build):
This should be merged with buildbot_commands.GenerateBreakpadSymbols()
once we rewrite cros_generate_breakpad_symbols in python.
Args:
board: The board whose symbols we wish to generate
breakpad_dir: The full path to the breakpad directory where symbols live
strip_cfi: Do not generate CFI data
generate_count: If set, only generate this many symbols (meant for testing)
sysroot: The root where to find the corresponding ELFs
Returns:
The number of errors that were encountered.
"""
if breakpad_dir is None:
breakpad_dir = FindBreakpadDir(board)
if sysroot is None:
sysroot = os.path.join('/build', board)
# Make sure non-root can write out symbols as needed.
osutils.SafeMakedirs(breakpad_dir, sudo=True)
if not os.access(breakpad_dir, os.W_OK):
cros_build_lib.SudoRunCommand(
['chown', '-R', str(os.getuid()), breakpad_dir])
debug_dir = FindDebugDir(board)
cros_build_lib.Info('generating all breakpad symbol files using %s',
debug_dir)
# Let's locate all the debug_files first and their size. This way we can
# start processing the largest files first in parallel with the small ones.
debug_files = []
for root, _, files in os.walk(debug_dir):
for debug_file in files:
debug_file = os.path.join(root, debug_file)
if debug_file.endswith('.ko.debug'):
cros_build_lib.Debug('Skipping kernel module %s', debug_file)
elif debug_file.endswith('.debug'):
if os.path.islink(debug_file):
# The build-id stuff is common enough to filter out by default.
if '/.build-id/' in debug_file:
msg = cros_build_lib.Debug
else:
msg = cros_build_lib.Warning
msg('Skipping symbolic link %s', debug_file)
else:
debug_files.append(
(os.path.getsize(debug_file), debug_file))
# Now start generating symbols for all the inputs.
bg_errors = multiprocessing.Value('i')
with parallel.BackgroundTaskRunner(GenerateBreakpadSymbol,
breakpad_dir=breakpad_dir,
board=board,
strip_cfi=strip_cfi,
num_errors=bg_errors) as queue:
for _, debug_file in sorted(debug_files, reverse=True):
if generate_count == 0:
break
# Turn /build/$BOARD/usr/lib/debug/sbin/foo.debug into
# /build/$BOARD/sbin/foo.
elf_file = os.path.join(sysroot, debug_file[len(debug_dir) + 1:-6])
if not os.path.exists(elf_file):
# Sometimes we filter out programs from /usr/bin but leave behind
# the .debug file.
cros_build_lib.Warning('Skipping missing %s', elf_file)
continue
queue.put([elf_file, debug_file])
if generate_count is not None:
generate_count -= 1
if generate_count == 0:
break
return bg_errors.value
def testGetLatestSHA1ForProject4Realz(self):
"""Verify we can check the latest hash from chromite."""
helper = self._GetHelper()
cros_build_lib.Info('The current sha1 on master for chromite is: %s' %
helper.GetLatestSHA1ForBranch('chromiumos/chromite',
'master'))
def GenerateBreakpadSymbol(elf_file,
debug_file=None,
breakpad_dir=None,
board=None,
strip_cfi=False,
num_errors=None):
"""Generate the symbols for |elf_file| using |debug_file|
Args:
elf_file: The file to dump symbols for
debug_file: Split debug file to use for symbol information
breakpad_dir: The dir to store the output symbol file in
board: If |breakpad_dir| is not specified, use |board| to find it
strip_cfi: Do not generate CFI data
num_errors: An object to update with the error count (needs a .value member)
Returns:
The number of errors that were encountered.
"""
if breakpad_dir is None:
breakpad_dir = FindBreakpadDir(board)
if num_errors is None:
num_errors = ctypes.c_int()
cmd_base = ['dump_syms']
if strip_cfi:
cmd_base += ['-c']
# Some files will not be readable by non-root (e.g. set*id /bin/su).
needs_sudo = not os.access(elf_file, os.R_OK)
def _DumpIt(cmd_args):
if needs_sudo:
run_command = cros_build_lib.SudoRunCommand
else:
run_command = cros_build_lib.RunCommand
return run_command(cmd_base + cmd_args,
redirect_stderr=True,
log_stdout_to_file=temp.name,
error_code_ok=True,
debug_level=logging.DEBUG)
def _CrashCheck(ret, msg):
if ret < 0:
cros_build_lib.PrintBuildbotStepWarnings()
cros_build_lib.Warning('dump_syms crashed with %s; %s',
osutils.StrSignal(-ret), msg)
osutils.SafeMakedirs(breakpad_dir)
with tempfile.NamedTemporaryFile(dir=breakpad_dir, bufsize=0) as temp:
if debug_file:
# Try to dump the symbols using the debug file like normal.
cmd_args = [elf_file, os.path.dirname(debug_file)]
result = _DumpIt(cmd_args)
if result.returncode:
# Sometimes dump_syms can crash because there's too much info.
# Try dumping and stripping the extended stuff out. At least
# this way we'll get the extended symbols. http://crbug.com/266064
_CrashCheck(result.returncode, 'retrying w/out CFI')
cmd_args = ['-c', '-r'] + cmd_args
result = _DumpIt(cmd_args)
_CrashCheck(result.returncode, 'retrying w/out debug')
basic_dump = result.returncode
else:
basic_dump = True
if basic_dump:
# If that didn't work (no debug, or dump_syms still failed), try
# dumping just the file itself directly.
result = _DumpIt([elf_file])
if result.returncode:
# A lot of files (like kernel files) contain no debug information,
# do not consider such occurrences as errors.
cros_build_lib.PrintBuildbotStepWarnings()
_CrashCheck(result.returncode, 'giving up entirely')
if 'file contains no debugging information' in result.error:
cros_build_lib.Warning('no symbols found for %s', elf_file)
else:
num_errors.value += 1
cros_build_lib.Error('dumping symbols for %s failed:\n%s',
elf_file, result.error)
return num_errors.value
# Move the dumped symbol file to the right place:
# /build/$BOARD/usr/lib/debug/breakpad/<module-name>/<id>/<module-name>.sym
header = ReadSymsHeader(temp)
cros_build_lib.Info('Dumped %s as %s : %s', elf_file, header.name,
header.id)
sym_file = os.path.join(breakpad_dir, header.name, header.id,
header.name + '.sym')
osutils.SafeMakedirs(os.path.dirname(sym_file))
os.rename(temp.name, sym_file)
os.chmod(sym_file, 0644)
temp.delete = False
return num_errors.value
def _Print(cls, message):
"""Verbose print function."""
if cls.VERBOSE:
cros_build_lib.Info(message)
def _GenerateBlameListSinceLKGM(self):
"""Prints out links to all CL's that have been committed since LKGM.
Add buildbot trappings to print <a href='url'>text</a> in the waterfall for
each CL committed since we last had a passing build.
"""
if not self._ShouldGenerateBlameListSinceLKGM():
logging.info(
'Not generating blamelist for lkgm as it is not appropriate '
'for this build type.')
return
handler = git.Manifest(self.lkgm_path)
reviewed_on_re = re.compile('\s*Reviewed-on:\s*(\S+)')
author_re = re.compile('\s*Author:.*<(\S+)@\S+>\s*')
committer_re = re.compile('\s*Commit:.*<(\S+)@\S+>\s*')
for project in handler.projects.keys():
rel_src_path = handler.projects[project].get('path')
# If it's not part of our source tree, it doesn't affect our build.
if not rel_src_path:
continue
# Additional case in case the repo has been removed from the manifest.
src_path = self.cros_source.GetRelativePath(rel_src_path)
if not os.path.exists(src_path):
cros_build_lib.Info('Detected repo removed from manifest %s' %
project)
continue
revision = handler.projects[project]['revision']
result = cros_build_lib.RunCommand(
['git', 'log', '--pretty=full',
'%s..HEAD' % revision],
print_cmd=False,
redirect_stdout=True,
cwd=src_path)
current_author = None
current_committer = None
for line in result.output.splitlines():
author_match = author_re.match(line)
if author_match:
current_author = author_match.group(1)
committer_match = committer_re.match(line)
if committer_match:
current_committer = committer_match.group(1)
review_match = reviewed_on_re.match(line)
if review_match:
review = review_match.group(1)
_, _, change_number = review.rpartition('/')
if current_committer != 'chrome-bot':
cros_build_lib.PrintBuildbotLink(
'CHUMP %s:%s' % (current_author, change_number),
review)
elif self.build_type != constants.PALADIN_TYPE:
# Suppress re-printing changes we tried ourselves on paladin
# builders since they are redundant.
cros_build_lib.PrintBuildbotLink(
'%s:%s' % (current_author, change_number), review)
def main(_argv):
parser = optparse.OptionParser('cros_mark_as_stable OPTIONS packages')
parser.add_option('--all',
action='store_true',
help='Mark all packages as stable.')
parser.add_option('-b',
'--boards',
default='',
help='Colon-separated list of boards')
parser.add_option('--drop_file',
help='File to list packages that were revved.')
parser.add_option('--dryrun',
action='store_true',
help='Passes dry-run to git push if pushing a change.')
parser.add_option('-o',
'--overlays',
help='Colon-separated list of overlays to modify.')
parser.add_option('-p',
'--packages',
help='Colon separated list of packages to rev.')
parser.add_option('-r',
'--srcroot',
default=os.path.join(constants.SOURCE_ROOT, 'src'),
help='Path to root src directory.')
parser.add_option('--verbose',
action='store_true',
help='Prints out debug info.')
(options, args) = parser.parse_args()
portage_utilities.EBuild.VERBOSE = options.verbose
if len(args) != 1:
_PrintUsageAndDie('Must specify a valid command [commit, push]')
command = args[0]
package_list = None
if options.packages:
package_list = options.packages.split(':')
_CheckSaneArguments(command, options)
if options.overlays:
overlays = {}
for path in options.overlays.split(':'):
if not os.path.isdir(path):
cros_build_lib.Die('Cannot find overlay: %s' % path)
overlays[path] = []
else:
cros_build_lib.Warning('Missing --overlays argument')
overlays = {
'%s/private-overlays/chromeos-overlay' % options.srcroot: [],
'%s/third_party/chromiumos-overlay' % options.srcroot: []
}
manifest = git.ManifestCheckout.Cached(options.srcroot)
if command == 'commit':
portage_utilities.BuildEBuildDictionary(overlays, options.all,
package_list)
# Contains the array of packages we actually revved.
revved_packages = []
new_package_atoms = []
# Slight optimization hack: process the chromiumos overlay before any other
# cros-workon overlay first so we can do background cache generation in it.
# A perfect solution would walk all the overlays, figure out any dependencies
# between them (with layout.conf), and then process them in dependency order.
# However, this operation isn't slow enough to warrant that level of
# complexity, so we'll just special case the main overlay.
#
# Similarly, generate the cache in the portage-stable tree asap. We know
# we won't have any cros-workon packages in there, so generating the cache
# is the only thing it'll be doing. The chromiumos overlay instead might
# have revbumping to do before it can generate the cache.
keys = overlays.keys()
for overlay in ('/third_party/chromiumos-overlay',
'/third_party/portage-stable'):
for k in keys:
if k.endswith(overlay):
keys.remove(k)
keys.insert(0, k)
break
with parallel.BackgroundTaskRunner(portage_utilities.RegenCache) as queue:
for overlay in keys:
ebuilds = overlays[overlay]
if not os.path.isdir(overlay):
cros_build_lib.Warning("Skipping %s" % overlay)
continue
# Note we intentionally work from the non push tracking branch;
# everything built thus far has been against it (meaning, http mirrors),
# thus we should honor that. During the actual push, the code switches
# to the correct urls, and does an appropriate rebasing.
tracking_branch = git.GetTrackingBranchViaManifest(
overlay, manifest=manifest)[1]
if command == 'push':
PushChange(constants.STABLE_EBUILD_BRANCH,
tracking_branch,
options.dryrun,
cwd=overlay)
elif command == 'commit':
existing_branch = git.GetCurrentBranch(overlay)
work_branch = GitBranch(constants.STABLE_EBUILD_BRANCH,
tracking_branch,
cwd=overlay)
work_branch.CreateBranch()
if not work_branch.Exists():
cros_build_lib.Die(
'Unable to create stabilizing branch in %s' % overlay)
# In the case of uprevving overlays that have patches applied to them,
# include the patched changes in the stabilizing branch.
if existing_branch:
cros_build_lib.RunCommand(
['git', 'rebase', existing_branch],
print_cmd=False,
cwd=overlay)
messages = []
for ebuild in ebuilds:
if options.verbose:
cros_build_lib.Info('Working on %s', ebuild.package)
try:
new_package = ebuild.RevWorkOnEBuild(
options.srcroot, manifest)
if new_package:
revved_packages.append(ebuild.package)
new_package_atoms.append('=%s' % new_package)
messages.append(_GIT_COMMIT_MESSAGE %
ebuild.package)
except (OSError, IOError):
cros_build_lib.Warning(
'Cannot rev %s\n' % ebuild.package +
'Note you will have to go into %s '
'and reset the git repo yourself.' % overlay)
raise
if messages:
portage_utilities.EBuild.CommitChange(
'\n\n'.join(messages), overlay)
if cros_build_lib.IsInsideChroot():
# Regenerate caches if need be. We do this all the time to
# catch when users make changes without updating cache files.
queue.put([overlay])
if command == 'commit':
if cros_build_lib.IsInsideChroot():
CleanStalePackages(options.boards.split(':'), new_package_atoms)
if options.drop_file:
osutils.WriteFile(options.drop_file, ' '.join(revved_packages))
