def testToTCompatibility(self):
"""Validate that ToT can use our pickles, and that we can use ToT's data."""
repo = os.path.join(self.tempdir, 'chromite')
reference = os.path.abspath(__file__)
reference = os.path.normpath(os.path.join(reference, '../../'))
repository.CloneGitRepo(repo,
'%s/chromiumos/chromite' %
constants.GIT_HTTP_URL,
reference=reference)
code = """
import sys
from chromite.buildbot import validation_pool_unittest
if not hasattr(validation_pool_unittest, 'TestPickling'):
sys.exit(0)
sys.stdout.write(validation_pool_unittest.TestPickling.%s)
"""
# Verify ToT can take our pickle.
cros_build_lib.RunCommandCaptureOutput(
['python', '-c', code % '_CheckTestData(sys.stdin.read())'],
cwd=self.tempdir,
print_cmd=False,
input=self._GetTestData())
# Verify we can handle ToT's pickle.
ret = cros_build_lib.RunCommandCaptureOutput(
['python', '-c', code % '_GetTestData()'],
cwd=self.tempdir,
print_cmd=False)
self._CheckTestData(ret.output)
def _DoCleanup(self):
"""Wipe unused repositories."""
# Find all projects, even if they're not in the manifest. Note the find
# trickery this is done to keep it as fast as possible.
repo_path = os.path.join(self.directory, '.repo', 'projects')
current = set(cros_build_lib.RunCommandCaptureOutput(
['find', repo_path, '-type', 'd', '-name', '*.git', '-printf', '%P\n',
'-a', '!', '-wholename', '*.git/*', '-prune'],
print_cmd=False).output.splitlines())
data = {}.fromkeys(current, 0)
path = os.path.join(self.directory, '.repo', 'project.lru')
if os.path.exists(path):
existing = [x.strip().split(None, 1)
for x in osutils.ReadFile(path).splitlines()]
data.update((k, int(v)) for k, v in existing if k in current)
# Increment it all...
data.update((k, v + 1) for k, v in data.iteritems())
# Zero out what is now used.
projects = git.ManifestCheckout.Cached(self.directory).projects
data.update(('%s.git' % x['path'], 0) for x in projects.itervalues())
# Finally... wipe anything that's greater than our threshold.
wipes = [k for k, v in data.iteritems() if v > self.LRU_THRESHOLD]
if wipes:
cros_build_lib.SudoRunCommand(
['rm', '-rf'] + [os.path.join(repo_path, proj) for proj in wipes])
map(data.pop, wipes)
osutils.WriteFile(path, "\n".join('%s %i' % x for x in data.iteritems()))
def ExportManifest(self, mark_revision=False, revisions=True):
"""Export the revision locked manifest
Args:
mark_revision: If True, then the sha1 of manifest.git is recorded
into the resultant manifest tag as a version attribute.
Specifically, if manifests.git is at 1234, <manifest> becomes
<manifest revision="1234">.
revisions: If True, then rewrite all branches/tags into a specific
sha1 revision. If False, don't.
Returns:
The manifest as a string.
"""
cmd = ['repo', 'manifest', '-o', '-']
if revisions:
cmd += ['-r']
output = cros_build_lib.RunCommandCaptureOutput(
cmd, cwd=self.directory, print_cmd=False,
extra_env={'PAGER':'cat'}).output
if not mark_revision:
return output
modified = git.RunGit(os.path.join(self.directory, '.repo/manifests'),
['rev-list', '-n1', 'HEAD'])
assert modified.output
return output.replace("<manifest>", '<manifest revision="%s">' %
modified.output.strip())
def GetNewestFileTime(path, ignore_subdirs=[]):
#pylint: disable-msg=W0102
"""Recursively determine the newest file modification time.
Arguments:
path: The absolute path of the directory to recursively search.
ignore_subdirs: list of names of subdirectores of given path, to be
ignored by recursive search. Useful as a speed
optimization, to ignore directories full of many
files.
Returns:
The modification time of the most recently modified file recursively
contained within the specified directory. Returned as seconds since
Jan. 1, 1970, 00:00 GMT, with fractional part (floating point number).
"""
command = ['find', path]
for ignore in ignore_subdirs:
command.extend(['-path', os.path.join(path, ignore), '-prune', '-o'])
command.extend(['-printf', r'%T@\n'])
command_result = cros_build_lib.RunCommandCaptureOutput(command,
error_code_ok=True)
float_times = [
float(str_time) for str_time in command_result.output.split('\n')
if str_time != ''
]
return max(float_times)
def testPushChange(self):
git_log = 'Marking test_one as stable\nMarking test_two as stable\n'
fake_description = 'Marking set of ebuilds as stable\n\n%s' % git_log
self.mox.StubOutWithMock(cros_mark_as_stable, '_DoWeHaveLocalCommits')
self.mox.StubOutWithMock(cros_mark_as_stable.GitBranch, 'CreateBranch')
self.mox.StubOutWithMock(cros_mark_as_stable.GitBranch, 'Exists')
self.mox.StubOutWithMock(git, 'PushWithRetry')
self.mox.StubOutWithMock(git, 'GetTrackingBranch')
self.mox.StubOutWithMock(git, 'SyncPushBranch')
self.mox.StubOutWithMock(git, 'CreatePushBranch')
self.mox.StubOutWithMock(git, 'RunGit')
cros_mark_as_stable._DoWeHaveLocalCommits(
self._branch, self._target_manifest_branch, '.').AndReturn(True)
git.GetTrackingBranch('.', for_push=True).AndReturn(
['gerrit', 'refs/remotes/gerrit/master'])
git.SyncPushBranch('.', 'gerrit', 'refs/remotes/gerrit/master')
cros_mark_as_stable._DoWeHaveLocalCommits(
self._branch, 'refs/remotes/gerrit/master', '.').AndReturn(True)
result = cros_build_lib.CommandResult(output=git_log)
cros_build_lib.RunCommandCaptureOutput(
['git', 'log', '--format=format:%s%n%n%b',
'refs/remotes/gerrit/master..%s' % self._branch],
cwd='.').AndReturn(result)
git.CreatePushBranch('merge_branch', '.')
git.RunGit('.', ['merge', '--squash', self._branch])
git.RunGit('.', ['commit', '-m', fake_description])
git.RunGit('.', ['config', 'push.default', 'tracking'])
git.PushWithRetry('merge_branch', '.', dryrun=False)
self.mox.ReplayAll()
cros_mark_as_stable.PushChange(self._branch, self._target_manifest_branch,
False, '.')
self.mox.VerifyAll()
def _run(self, cmd, cwd=None):
# Note that cwd is intentionally set to a location the user can't write
# to; this flushes out any bad usage in the tests that would work by
# fluke of being invoked from w/in a git repo.
if cwd is None:
cwd = self.default_cwd
return cros_build_lib.RunCommandCaptureOutput(
cmd, cwd=cwd, print_cmd=False).output.strip()
def _GomaPort(self, goma_dir):
"""Returns current active Goma port."""
port = cros_build_lib.RunCommandCaptureOutput(
self.GOMACC_PORT_CMD,
cwd=goma_dir,
debug_level=logging.DEBUG,
error_code_ok=True).output.strip()
return port
def testCheckoutCreate(self):
# Test init with no previous branch existing.
self._branch.Exists(self._branch_name).AndReturn(False)
cros_build_lib.RunCommandCaptureOutput(['repo', 'start', self._branch_name,
'.'], print_cmd=False, cwd='.')
self.mox.ReplayAll()
cros_mark_as_stable.GitBranch.Checkout(self._branch)
self.mox.VerifyAll()
def Exists(self, branch=None):
"""Returns True if the branch exists."""
if not branch:
branch = self.branch_name
branches = cros_build_lib.RunCommandCaptureOutput(['git', 'branch'],
print_cmd=False,
cwd=self.cwd).output
return branch in branches.split()
def setUp(self):
self.manifest_dir = os.path.join(self.tempdir, '.repo', 'manifests')
# Initialize a repo intance here.
# TODO(vapier, ferringb): mangle this so it inits from a local
# checkout if one is available, same for the git-repo fetch.
cmd = ['repo', 'init', '-u', constants.MANIFEST_URL]
cros_build_lib.RunCommandCaptureOutput(cmd, cwd=self.tempdir, input='')
self.active_manifest = os.path.realpath(
os.path.join(self.tempdir, '.repo', 'manifest.xml'))
def testInferBuildRootExists(self):
"""Test that we don't prompt the user if buildroot already exists."""
cros_build_lib.RunCommandCaptureOutput(['touch', self.external_marker])
os.utime(self.external_marker, None)
(cros_build_lib.GetInput(mox.IgnoreArg()).InAnyOrder().AndRaise(
TestFailedException()))
self.mox.ReplayAll()
self.assertMain(['--local', 'x86-generic-paladin'])
def SymUpload(sym_file, upload_url):
"""Run breakpad sym_upload helper"""
# TODO(vapier): Rewrite to use native python HTTP libraries. This tool
# reads the sym_file and does a HTTP post to URL with a few fields set.
# See the tiny breakpad/tools/linux/symupload/sym_upload.cc for details.
cmd = ['sym_upload', sym_file, upload_url]
with cros_build_lib.SubCommandTimeout(UPLOAD_TIMEOUT):
return cros_build_lib.RunCommandCaptureOutput(
cmd, debug_level=logging.DEBUG)
def ParseBashArray(value):
"""Parse a valid bash array into python list."""
# The syntax for bash arrays is nontrivial, so let's use bash to do the
# heavy lifting for us.
sep = ','
# Because %s may contain bash comments (#), put a clever newline in the way.
cmd = 'ARR=%s\nIFS=%s; echo -n "${ARR[*]}"' % (value, sep)
return cros_build_lib.RunCommandCaptureOutput(
cmd, print_cmd=False, shell=True).output.split(sep)
def testDepotTools(self):
"""Test that the entry point used by depot_tools works."""
path = os.path.join(constants.SOURCE_ROOT, 'chromite', 'buildbot',
'cbuildbot')
# Verify the tests below actually are testing correct behaviour;
# specifically that it doesn't always just return 0.
self.assertRaises(cros_build_lib.RunCommandError,
cros_build_lib.RunCommandCaptureOutput,
['cbuildbot', '--monkeys'],
cwd=constants.SOURCE_ROOT)
# Validate depot_tools lookup.
cros_build_lib.RunCommandCaptureOutput(['cbuildbot', '--help'],
cwd=constants.SOURCE_ROOT)
# Validate buildbot invocation pathway.
cros_build_lib.RunCommandCaptureOutput([path, '--help'],
cwd=constants.SOURCE_ROOT)
def testExists(self):
branch = cros_mark_as_stable.GitBranch(self._branch_name,
self._target_manifest_branch, '.')
# Test if branch exists that is created
result = cros_build_lib.CommandResult(output=self._branch_name + '\n')
cros_build_lib.RunCommandCaptureOutput(['git', 'branch'], print_cmd=False,
cwd='.').AndReturn(result)
self.mox.ReplayAll()
self.assertTrue(branch.Exists())
self.mox.VerifyAll()
def Checkout(self, branch=None):
"""Function used to check out to another GitBranch."""
if not branch:
branch = self.branch_name
if branch == self.tracking_branch or self.Exists(branch):
git_cmd = ['git', 'checkout', '-f', branch]
else:
git_cmd = ['repo', 'start', branch, '.']
cros_build_lib.RunCommandCaptureOutput(git_cmd, print_cmd=False,
cwd=self.cwd)
def ListWorkonPackages(board, host):
"""List the packages that are currently being worked on.
Args:
board: The board to look at. If host is True, this should be set to None.
host: Whether to look at workon packages for the host.
"""
cmd = [os.path.join(constants.CROSUTILS_DIR, 'cros_workon'), 'list']
cmd.extend(['--host'] if host else ['--board', board])
result = cros_build_lib.RunCommandCaptureOutput(cmd, print_cmd=False)
return result.output.split()
def version(self):
obj = self._version
if obj is None:
# We suppress the gerrit version call's logging; it's basically
# never useful log wise.
obj = cros_build_lib.RunCommandCaptureOutput(
self.ssh_prefix + ['gerrit', 'version'],
print_cmd=False).output.strip()
obj = obj.replace('gerrit version ', '')
self._version = obj
return obj
def _LastModificationTime(self, path):
"""Calculate the last time a directory subtree was modified.
Args:
path: Directory to look at.
"""
cmd = 'find . -name .git -prune -o -printf "%T@\n" | sort -nr | head -n1'
ret = cros_build_lib.RunCommandCaptureOutput(cmd,
cwd=path,
shell=True,
print_cmd=False)
return float(ret.output) if ret.output else 0
def _AddProjectsToManifestGroups(options, *projects):
"""Enable the given manifest groups for the configured repository."""
groups_to_enable = ['name:%s' % x for x in projects]
git_config = options.git_config
enabled_groups = cros_build_lib.RunCommandCaptureOutput(
['git', 'config', '-f', git_config, '--get', 'manifest.groups'],
error_code_ok=True,
print_cmd=False).output.split(',')
# Note that ordering actually matters, thus why the following code
# is written this way.
# Per repo behaviour, enforce an appropriate platform group if
# we're converting from a default manifest group to a limited one.
# Finally, note we reprocess the existing groups; this is to allow
# us to cleanup any user screwups, or our own screwups.
requested_groups = (
['minilayout',
'platform-%s' % (platform.system().lower(), )] + enabled_groups +
list(groups_to_enable))
processed_groups = set()
finalized_groups = []
for group in requested_groups:
if group not in processed_groups:
finalized_groups.append(group)
processed_groups.add(group)
cros_build_lib.RunCommandCaptureOutput([
'git', 'config', '-f', git_config, 'manifest.groups',
','.join(finalized_groups)
],
print_cmd=False)
def RemoteSh(self,
cmd,
connect_settings=None,
error_code_ok=False,
ssh_error_ok=False,
debug_level=None):
"""Run a sh command on the remote device through ssh.
Arguments:
cmd: The command string or list to run.
connect_settings: The SSH connect settings to use.
error_code_ok: Does not throw an exception when the command exits with a
non-zero returncode. This does not cover the case where
the ssh command itself fails (return code 255).
See ssh_error_ok.
ssh_error_ok: Does not throw an exception when the ssh command itself
fails (return code 255).
debug_level: See cros_build_lib.RunCommand documentation.
Returns:
A CommandResult object. The returncode is the returncode of the command,
or 255 if ssh encountered an error (could not connect, connection
interrupted, etc.)
Raises: RunCommandError when error is not ignored through error_code_ok and
ssh_error_ok flags.
"""
if not debug_level:
debug_level = self.debug_level
ssh_cmd = self._GetSSHCmd(connect_settings)
if isinstance(cmd, basestring):
ssh_cmd += [self.target_ssh_url, '--', cmd]
else:
ssh_cmd += [self.target_ssh_url, '--'] + cmd
try:
result = cros_build_lib.RunCommandCaptureOutput(
ssh_cmd, debug_level=debug_level)
except cros_build_lib.RunCommandError as e:
if ((e.result.returncode == SSH_ERROR_CODE and ssh_error_ok) or
(e.result.returncode and e.result.returncode != SSH_ERROR_CODE
and error_code_ok)):
result = e.result
else:
raise
return result
def BestVisible(atom, board=None, buildroot=constants.SOURCE_ROOT):
"""Get the best visible ebuild CPV for the given atom.
Args:
atom: Portage atom.
board: Board to look at. By default, look in chroot.
root: Directory
Returns:
A CPV object.
"""
portageq = 'portageq' if board is None else 'portageq-%s' % board
root = '/' if board is None else '/build/%s' % board
cmd = [portageq, 'best_visible', root, 'ebuild', atom]
result = cros_build_lib.RunCommandCaptureOutput(
cmd, cwd=buildroot, enter_chroot=True, debug_level=logging.DEBUG)
return SplitCPV(result.output.strip())
def testGetCommitId(self):
fake_sources = '/path/to/sources'
fake_hash = '24ab3c9f6d6b5c744382dba2ca8fb444b9808e9f'
fake_ebuild_path = '/path/to/test_package/test_package-9999.ebuild'
fake_ebuild = self._makeFakeEbuild(fake_ebuild_path)
# git rev-parse HEAD
self.mox.StubOutWithMock(cros_build_lib, 'RunCommandCaptureOutput')
result = _DummyCommandResult(fake_hash)
cros_build_lib.RunCommandCaptureOutput(
mox.IgnoreArg(),
cwd=mox.IgnoreArg(),
print_cmd=portage_utilities.EBuild.VERBOSE).AndReturn(result)
self.mox.ReplayAll()
test_hash = fake_ebuild.GetCommitId(fake_sources)
self.mox.VerifyAll()
self.assertEquals(test_hash, fake_hash)
def FindParent(host, port, name):
"""Find the parent of a given project."""
lines = cros_build_lib.RunCommandCaptureOutput([
'ssh', host, '-p', port, 'gerrit', 'ls-projects', '--type',
'PERMISSIONS'
],
shell=False).output
# Normalize the output to protect ourselves against any gerrit bugs.
projects = [os.path.normpath(x) for x in lines.splitlines() if x]
# Add a '/' to the project so that we consider matches at the directory
# level, rather than intersecting 'clank' against 'clank-bot'.
scored = sorted((x for x in projects if name.startswith(x + '/')), key=len)
if scored:
# Use the project with the longest common prefix.
return scored[-1]
# Since we only handle chromeos and chromiumos projects, we should always
# find a valid parent project. Raise an error if this isn't the case.
raise AssertionError('Missing parent project for %s' % name)
def _InstallBuildDependencies(self):
# Calculate buildtime deps that are not runtime deps.
raw_sysroot = '/build/%s' % (self.options.board, )
cmd = ['qdepends', '-q', '-C', self.options.package]
output = cros_build_lib.RunCommandCaptureOutput(cmd,
extra_env={
'ROOT': raw_sysroot
}).output
if output.count('\n') > 1:
raise AssertionError(
'Too many packages matched given package pattern')
# qdepend outputs "package: deps", so only grab the deps.
atoms = output.partition(':')[2].split()
# Install the buildtime deps.
if atoms:
self._Emerge(*atoms)
def PushChange(stable_branch, tracking_branch, dryrun, cwd):
"""Pushes commits in the stable_branch to the remote git repository.
Pushes local commits from calls to CommitChange to the remote git
repository specified by current working directory. If changes are
found to commit, they will be merged to the merge branch and pushed.
In that case, the local repository will be left on the merge branch.
Args:
stable_branch: The local branch with commits we want to push.
tracking_branch: The tracking branch of the local branch.
dryrun: Use git push --dryrun to emulate a push.
cwd: The directory to run commands in.
Raises:
OSError: Error occurred while pushing.
"""
if not _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd):
cros_build_lib.Info('No work found to push in %s. Exiting', cwd)
return
# For the commit queue, our local branch may contain commits that were
# just tested and pushed during the CommitQueueCompletion stage. Sync
# and rebase our local branch on top of the remote commits.
remote, push_branch = git.GetTrackingBranch(cwd, for_push=True)
git.SyncPushBranch(cwd, remote, push_branch)
# Check whether any local changes remain after the sync.
if not _DoWeHaveLocalCommits(stable_branch, push_branch, cwd):
cros_build_lib.Info('All changes already pushed for %s. Exiting', cwd)
return
description = cros_build_lib.RunCommandCaptureOutput([
'git', 'log', '--format=format:%s%n%n%b',
'%s..%s' % (push_branch, stable_branch)
],
cwd=cwd).output
description = 'Marking set of ebuilds as stable\n\n%s' % description
cros_build_lib.Info('For %s, using description %s', cwd, description)
git.CreatePushBranch(constants.MERGE_BRANCH, cwd)
git.RunGit(cwd, ['merge', '--squash', stable_branch])
git.RunGit(cwd, ['commit', '-m', description])
git.RunGit(cwd, ['config', 'push.default', 'tracking'])
git.PushWithRetry(constants.MERGE_BRANCH, cwd, dryrun=dryrun)
def Run(self, tool, args, cwd=None, sudo=False):
"""Run a tool with given arguments.
The tool name may be used unchanged or substituted with a full path if
required.
The tool and arguments can use ##/ to signify the chroot (at the beginning
of the tool/argument).
Args:
tool: Name of tool to run.
args: List of arguments to pass to tool.
cwd: Directory to change into before running tool (None if none).
sudo: True to run the tool with sudo
Returns:
Output of tool (stdout).
Raises:
CmdError: If running the tool, or the tool itself creates an error.
"""
if tool in self._tools:
tool = self._tools[tool]
tool = self.Filename(tool)
args = [self.Filename(arg) for arg in args]
cmd = [tool] + args
if sudo:
cmd.insert(0, 'sudo')
try:
result = cros_build_lib.RunCommandCaptureOutput(
cmd,
cwd=cwd,
print_cmd=self._out.verbose > 3,
combine_stdout_stderr=True,
error_code_ok=True)
except cros_build_lib.RunCommandError as ex:
raise CmdError(str(ex))
stdout = result.output
if result.returncode:
raise CmdError('Command failed: %s\n%s' % (' '.join(cmd), stdout))
self._out.Debug(stdout)
return stdout
def GetWorkonProjectMap(overlay, subdirectories):
"""Get the project -> ebuild mapping for cros_workon ebuilds.
Args:
overlay: Overlay to look at.
subdirectories: List of subdirectories to look in on the overlay.
Returns:
A list of (filename, projects) tuples for cros-workon ebuilds in the
given overlay under the given subdirectories.
"""
# Search ebuilds for project names, ignoring non-existent directories.
cmd = ['grep', '^CROS_WORKON_PROJECT=', '--include', '*-9999.ebuild',
'-Hsr'] + list(subdirectories)
result = cros_build_lib.RunCommandCaptureOutput(
cmd, cwd=overlay, error_code_ok=True, print_cmd=False)
for grep_line in result.output.splitlines():
filename, _, line = grep_line.partition(':')
value = line.partition('=')[2]
projects = ParseBashArray(value)
yield filename, projects
def RunGit(git_repo, cmd, **kwds):
"""RunCommandCaptureOutput wrapper for git commands.
This suppresses print_cmd, and suppresses output by default. Git
functionality w/in this module should use this unless otherwise
warranted, to standardize git output (primarily, keeping it quiet
and being able to throw useful errors for it).
Args:
git_repo: Pathway to the git repo to operate on.
cmd: A sequence of the git subcommand to run. The 'git' prefix is
added automatically. If you wished to run 'git remote update',
this would be ['remote', 'update'] for example.
kwds: Any RunCommand options/overrides to use.
Returns:
A CommandResult object."""
kwds.setdefault('print_cmd', False)
cros_build_lib.Debug("RunGit(%r, %r, **%r)", git_repo, cmd, kwds)
return cros_build_lib.RunCommandCaptureOutput(['git'] + cmd,
cwd=git_repo,
**kwds)
def SetReviewers(self, change, add=(), remove=(), project=None):
"""Adjust the reviewers list for a given change.
Arguments:
change: Either the ChangeId, or preferably, the gerrit change number.
If you use a ChangeId be aware that this command will fail if multiple
changes match. Can be either a string or an integer.
add: Either this or removes must be given. If given, it must be a
either a single email address/group name, or a sequence of email
addresses or group names to add as reviewers. Note it's not
considered an error if you attempt to add a reviewer that
already is marked as a reviewer for the change.
remove: Same rules as 'add', just is the list of reviewers to remove.
project: If given, the project to find the given change w/in. Unnecessary
if passing a gerrit number for change; if passing a ChangeId, strongly
advised that a project be specified.
Raises:
RunCommandError if the attempt to modify the reviewers list fails. If
the command fails, no changes to the reviewer list occurs.
"""
if not add and not remove:
raise ValueError('Either add or remove must be non empty')
command = self.ssh_prefix + ['gerrit', 'set-reviewers']
command.extend(
cros_build_lib.iflatten_instance(
[('--add', x) for x in cros_build_lib.iflatten_instance(add)] +
[('--remove', x)
for x in cros_build_lib.iflatten_instance(remove)]))
if project is not None:
command += ['--project', project]
# Always set the change last; else --project may not take hold by that
# point, with gerrit complaining of duplicates when there aren't.
# Yes kiddies, gerrit can be retarded; this being one of those cases.
command.append(str(change))
cros_build_lib.RunCommandCaptureOutput(command,
print_cmd=self.print_cmd)