def testRemoteTrybotTrue(self):
"""Test GetBaseUploadURI with no archive base but remote_trybot is True."""
expected_result = (
'%s/trybot-%s' %
(config_lib.GetConfig().params.ARCHIVE_URL, DEFAULT_BOT_NAME))
result = self._GetBaseUploadURI(remote_trybot=True)
self.assertEqual(expected_result, result)
def GetBaseUploadURI(config, archive_base=None, bot_id=None,
remote_trybot=False):
"""Get the base URL where artifacts from this builder are uploaded.
Each build run stores its artifacts in a subdirectory of the base URI.
We also have LATEST files under the base URI which help point to the
latest build available for a given builder.
Args:
config: The build config to examine.
archive_base: Optional. The root URL under which objects from all
builders are uploaded. If not specified, we use the default archive
bucket.
bot_id: The bot ID to archive files under.
remote_trybot: Whether this is a remote trybot run. This is used to
make sure that uploads from remote trybot runs do not conflict with
uploads from production builders.
Returns:
Google Storage URI (i.e. 'gs://...') under which all archived files
should be uploaded. In other words, a path like a directory, even
through GS has no real directories.
"""
if not bot_id:
bot_id = config.GetBotId(remote_trybot=remote_trybot)
if archive_base:
return '%s/%s' % (archive_base, bot_id)
elif remote_trybot or config.gs_path == config_lib.GS_PATH_DEFAULT:
return '%s/%s' % (config_lib.GetConfig().params.ARCHIVE_URL, bot_id)
else:
return config.gs_path
def main(argv):
cros_build_lib.AssertInsideChroot()
opts = _ParseArguments(argv)
site_config = config_lib.GetConfig()
logging.info('Generating board configs. This takes about 2m...')
for key in sorted(binhost.GetChromePrebuiltConfigs(site_config)):
binhost.GenConfigsForBoard(key.board,
regen=opts.regen,
error_code_ok=True)
# Fetch all compat IDs.
fetcher = binhost.CompatIdFetcher()
keys = binhost.GetChromePrebuiltConfigs(site_config).keys()
compat_ids = fetcher.FetchCompatIds(keys)
# Save the PFQ configs.
pfq_configs = binhost.PrebuiltMapping.Get(keys, compat_ids)
filename_internal = binhost.PrebuiltMapping.GetFilename(
opts.buildroot, 'chrome')
pfq_configs.Dump(filename_internal)
git.AddPath(filename_internal)
git.Commit(os.path.dirname(filename_internal),
'Update PFQ config dump',
allow_empty=True)
filename_external = binhost.PrebuiltMapping.GetFilename(opts.buildroot,
'chromium',
internal=False)
pfq_configs.Dump(filename_external, internal=False)
git.AddPath(filename_external)
git.Commit(os.path.dirname(filename_external),
'Update PFQ config dump',
allow_empty=True)
def testDefaultGSPathRemoteTrybotFalse(self):
"""Test GetBaseUploadURI with default gs_path value in config."""
self.cfg = _ExtendDefaultConfig(gs_path=config_lib.GS_PATH_DEFAULT)
# Test without bot_id.
expected_result = (
'%s/%s' %
(config_lib.GetConfig().params.ARCHIVE_URL, DEFAULT_BOT_NAME))
result = self._GetBaseUploadURI(remote_trybot=False)
self.assertEqual(expected_result, result)
# Test with bot_id.
expected_result = (
'%s/%s' % (config_lib.GetConfig().params.ARCHIVE_URL, self.BOT_ID))
result = self._GetBaseUploadURI(bot_id=self.BOT_ID,
remote_trybot=False)
self.assertEqual(expected_result, result)
def __init__(self,
cache_dir,
board,
clear_cache=False,
chrome_src=None,
sdk_path=None,
toolchain_path=None,
silent=False,
use_external_config=None):
"""Initialize the class.
Args:
cache_dir: The toplevel cache dir to use.
board: The board to manage the SDK for.
clear_cache: Clears the sdk cache during __init__.
chrome_src: The location of the chrome checkout. If unspecified, the
cwd is presumed to be within a chrome checkout.
sdk_path: The path (whether a local directory or a gs:// path) to fetch
SDK components from.
toolchain_path: The path (whether a local directory or a gs:// path) to
fetch toolchain components from.
silent: If set, the fetcher prints less output.
use_external_config: When identifying the configuration for a board,
force usage of the external configuration if both external and internal
are available.
"""
site_config = config_lib.GetConfig()
self.cache_base = os.path.join(cache_dir, COMMAND_NAME)
if clear_cache:
logging.warning('Clearing the SDK cache.')
osutils.RmDir(self.cache_base, ignore_missing=True)
self.tarball_cache = cache.TarballCache(
os.path.join(self.cache_base, self.TARBALL_CACHE))
self.misc_cache = cache.DiskCache(
os.path.join(self.cache_base, self.MISC_CACHE))
self.board = board
self.config = site_config.FindCanonicalConfigForBoard(
board, allow_internal=not use_external_config)
self.gs_base = archive_lib.GetBaseUploadURI(self.config)
self.clear_cache = clear_cache
self.chrome_src = chrome_src
self.sdk_path = sdk_path
self.toolchain_path = toolchain_path
self.silent = silent
# For external configs, there is no need to run 'gsutil config', because
# the necessary files are all accessible to anonymous users.
internal = self.config['internal']
self.gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=internal)
if self.sdk_path is None:
self.sdk_path = os.environ.get(self.SDK_PATH_ENV)
if self.toolchain_path is None:
self.toolchain_path = 'gs://%s' % constants.SDK_GS_BUCKET
def testGetConfigCaching(self):
"""Test that config_lib.GetConfig() caches it's results correctly."""
config_a = config_lib.GetConfig()
config_b = config_lib.GetConfig()
# Ensure that we get a SiteConfig, and that the result is cached.
self.assertIsInstance(config_a, config_lib.SiteConfig)
self.assertIs(config_a, config_b)
# Clear our cache.
config_lib.ClearConfigCache()
config_c = config_lib.GetConfig()
config_d = config_lib.GetConfig()
# Ensure that this gives us a new instance of the SiteConfig.
self.assertIsNot(config_a, config_c)
# But also that it's cached going forward.
self.assertIsInstance(config_c, config_lib.SiteConfig)
self.assertIs(config_c, config_d)
def main(argv):
opts = _ParseArguments(argv)
site_config = config_lib.GetConfig()
layout = {}
for config_name, config in site_config.iteritems():
active_waterfall = config['active_waterfall']
if not active_waterfall:
continue
waterfall_layout = layout.setdefault(active_waterfall, {})
board_layout = waterfall_layout[config_name] = {
'name': config_name,
'buildslave_type': config['buildslave_type'],
}
children = config['child_configs']
if children:
board_layout['children'] = [c['name'] for c in children]
opts.format(layout, sys.stdout)
import re
import shutil
import tempfile
from chromite.cbuildbot import config_lib
from chromite.cbuildbot import constants
from chromite.cbuildbot import repository
from chromite.cli import command
from chromite.lib import cros_build_lib
from chromite.lib import git
from chromite.lib import osutils
from chromite.lib import path_util
from chromite.lib import portage_util
from chromite.scripts import cros_mark_as_stable
site_config = config_lib.GetConfig()
class UprevNotFound(Exception):
"""Exception to throw when no Chrome Uprev CL is found."""
# git utility functions.
def CloneWorkingRepo(dest, url, reference, branch):
"""Clone a git repository with an existing local copy as a reference.
Also copy the hooks into the new repository.
Args:
def testBotIdRemoteTrybotTrue(self):
expected_result = (
'%s/%s' % (config_lib.GetConfig().params.ARCHIVE_URL, self.BOT_ID))
result = self._GetBaseUploadURI(bot_id=self.BOT_ID, remote_trybot=True)
self.assertEqual(expected_result, result)
def __init__(self, checkout_dir, dryrun): self._checkout_dir = checkout_dir self._dryrun = dryrun self._lkgm = None self._old_lkgm = None self.site_config = config_lib.GetConfig()
class PrebuiltCompatibilityTest(cros_test_lib.TestCase):
"""Ensure that prebuilts are present for all builders and are compatible."""
# Whether to cache setup from run to run. If set, requires that you install
# joblib (sudo easy_install joblib). This is useful for iterating on the
# unit tests, but note that if you 'repo sync', you'll need to clear out
# /tmp/joblib and blow away /build in order to update the caches. Note that
# this is never normally set to True -- if you want to use this feature,
# you'll need to hand-edit this file.
# TODO(davidjames): Add a --caching option.
CACHING = False
# A dict mapping BoardKeys to their associated compat ids.
COMPAT_IDS = None
# Boards that don't have Chromium PFQs.
# TODO(davidjames): Empty this list.
BOARDS_WITHOUT_CHROMIUM_PFQS = ['rush_ryu', 'smaug', 'veyron_rialto']
site_config = config_lib.GetConfig()
@classmethod
def setUpClass(cls):
assert cros_build_lib.IsInsideChroot()
logging.info('Generating board configs. This takes about 30m...')
board_keys = binhost.GetAllImportantBoardKeys(cls.site_config)
boards = set(key.board for key in board_keys)
for board in sorted(boards):
binhost.GenConfigsForBoard(board,
regen=not cls.CACHING,
error_code_ok=False)
fetcher = binhost.CompatIdFetcher(caching=cls.CACHING)
cls.COMPAT_IDS = fetcher.FetchCompatIds(list(board_keys))
def setUp(self):
self.complaints = []
self.fatal_complaints = []
def tearDown(self):
if self.complaints:
warnings.warn('\n' + '\n'.join(self.complaints))
if self.fatal_complaints:
self.assertFalse(self.fatal_complaints,
'\n'.join(self.fatal_complaints))
def Complain(self, msg, fatal):
"""Complain about an error when the test exits.
Args:
msg: The message to print.
fatal: Whether the message should be fatal. If not, the message will be
considered a warning.
"""
if fatal:
self.fatal_complaints.append(msg)
else:
self.complaints.append(msg)
def GetCompatIdDiff(self, expected, actual):
"""Return a string describing the differences between expected and actual.
Args:
expected: Expected value for CompatId.
actual: Actual value for CompatId.
"""
if expected.arch != actual.arch:
return 'arch differs: %s != %s' % (expected.arch, actual.arch)
elif expected.useflags != actual.useflags:
msg = self.GetSequenceDiff(expected.useflags, actual.useflags)
return msg.replace('Sequences', 'useflags')
elif expected.cflags != actual.cflags:
msg = self.GetSequenceDiff(expected.cflags, actual.cflags)
return msg.replace('Sequences', 'cflags')
else:
assert expected == actual
return 'no differences'
def AssertChromePrebuilts(self, pfq_configs, config, skip_useflags=False):
"""Verify that the specified config has Chrome prebuilts.
Args:
pfq_configs: A PrebuiltMapping object.
config: The config to check.
skip_useflags: Don't use extra useflags from the config.
"""
# Skip over useflags from the useflag if needed.
msg_prefix = ''
if skip_useflags and config.useflags:
msg_prefix = 'When we take out config-requested useflags (%s), ' % (
config.useflags)
config = config.deepcopy()
config.useflags = []
compat_id = self.GetCompatId(config)
pfqs = pfq_configs.by_compat_id.get(compat_id, set())
if not pfqs:
arch_useflags = (compat_id.arch, compat_id.useflags)
for key in pfq_configs.by_arch_useflags[arch_useflags]:
# If there wasn't an exact match for this CompatId, but there
# was an (arch, useflags) match, then we'll be using mismatched
# Chrome prebuilts. Complain.
# TODO(davidjames): This should be a fatal error for important
# builders, but we need to clean up existing cases first.
pfq_compat_id = self.COMPAT_IDS.get(key)
if pfq_compat_id:
err = self.GetCompatIdDiff(compat_id, pfq_compat_id)
msg = '%s%s uses mismatched Chrome prebuilts from %s -- %s'
self.Complain(msg %
(msg_prefix, config.name, key.board, err),
fatal=False)
pfqs.add(key)
if not pfqs:
pre_cq = (config.build_type == config_lib.CONFIG_TYPE_PRECQ)
msg = '%s%s cannot find Chrome prebuilts -- %s'
self.Complain(msg % (msg_prefix, config.name, compat_id),
fatal=pre_cq or config.important)
def GetCompatId(self, config, board=None):
"""Get the CompatId for a config.
Args:
config: A config_lib.BuildConfig object.
board: Board to use. Defaults to the first board in the config.
Optional if len(config.boards) == 1.
"""
if board is None:
assert len(config.boards) == 1
board = config.boards[0]
else:
assert board in config.boards
board_key = binhost.GetBoardKey(config, board)
compat_id = self.COMPAT_IDS.get(board_key)
if compat_id is None:
compat_id = binhost.CalculateCompatId(board, config.useflags)
self.COMPAT_IDS[board_key] = compat_id
return compat_id
def testChromePrebuiltsPresent(self, filename=None):
"""Verify Chrome prebuilts exist for all configs that build Chrome.
Args:
filename: Filename to load our PFQ mappings from. By default, generate
the PFQ mappings based on the current config.
"""
if filename is not None:
pfq_configs = binhost.PrebuiltMapping.Load(filename)
else:
keys = binhost.GetChromePrebuiltConfigs(self.site_config).keys()
pfq_configs = binhost.PrebuiltMapping.Get(keys, self.COMPAT_IDS)
for compat_id, pfqs in pfq_configs.by_compat_id.items():
if len(pfqs) > 1:
msg = 'The following Chrome PFQs produce identical prebuilts: %s -- %s'
self.Complain(msg % (', '.join(str(x)
for x in pfqs), compat_id),
fatal=False)
for _name, config in sorted(self.site_config.items()):
# Skip over configs that don't have Chrome or have >1 board.
if config.sync_chrome is False or len(config.boards) != 1:
continue
# Look for boards with missing prebuilts.
pre_cq = (config.build_type == config_lib.CONFIG_TYPE_PRECQ)
if ((config.usepkg_build_packages and not config.chrome_rev)
and (config.active_waterfall or pre_cq)):
self.AssertChromePrebuilts(pfq_configs, config)
# Check that we have a builder for the version w/o custom useflags as
# well.
if (config.useflags and config.boards[0]
not in self.BOARDS_WITHOUT_CHROMIUM_PFQS):
self.AssertChromePrebuilts(pfq_configs,
config,
skip_useflags=True)
def testCurrentChromePrebuiltsEnough(self):
"""Verify Chrome prebuilts exist for all configs that build Chrome.
This loads the list of Chrome prebuilts that were generated during the last
Chrome PFQ run from disk and verifies that it is sufficient.
"""
filename = binhost.PrebuiltMapping.GetFilename(constants.SOURCE_ROOT,
'chrome')
if os.path.exists(filename):
self.testChromePrebuiltsPresent(filename)
def testReleaseGroupSharing(self):
"""Verify that the boards built in release groups have compatible settings.
This means that all of the subconfigs in the release group have matching
use flags, cflags, and architecture.
"""
for config in self.site_config.values():
# Only test release groups.
if not config.name.endswith('-release-group'):
continue
# Get a list of the compatibility IDs.
compat_ids_for_config = collections.defaultdict(set)
for subconfig in config.child_configs:
if subconfig.sync_chrome is not False:
for board in subconfig.boards:
compat_id = self.GetCompatId(subconfig, board)
compat_ids_for_config[compat_id].add(board)
if len(compat_ids_for_config) > 1:
arch_useflags = set(
tuple(x[:-1]) for x in compat_ids_for_config)
if len(arch_useflags) > 1:
# If two configs in the same group have mismatched Chrome binaries
# (e.g. different use flags), Chrome may be built twice in parallel
# and this may result in flaky, slow, and possibly incorrect builds.
msg = '%s: %s and %s have mismatched Chrome binaries -- %s'
fatal = True
else:
# TODO(davidjames): This should be marked fatal once the
# ivybridge-freon-release-group is cleaned up.
msg = '%s: %s and %s have mismatched cflags -- %s'
fatal = False
ids, board_sets = zip(*compat_ids_for_config.iteritems())
boards = [next(iter(x)) for x in board_sets]
err = self.GetCompatIdDiff(ids[0], ids[1])
msg %= (config.name, boards[0], boards[1], err)
self.Complain(msg, fatal=fatal)
def testDumping(self):
"""Verify Chrome prebuilts exist for all configs that build Chrome.
This loads the list of Chrome prebuilts that were generated during the last
Chrome PFQ run from disk and verifies that it is sufficient.
"""
with osutils.TempDir() as tempdir:
keys = binhost.GetChromePrebuiltConfigs(self.site_config).keys()
pfq_configs = binhost.PrebuiltMapping.Get(keys, self.COMPAT_IDS)
filename = os.path.join(tempdir, 'foo.json')
pfq_configs.Dump(filename)
self.assertEqual(pfq_configs,
binhost.PrebuiltMapping.Load(filename))
