def GetParser():
"""Returns parser for ChromeCommitter.
Returns:
Parser for ChromeCommitter.
"""
# We need to use the account used by the builder to upload git CLs when
# generating CLs.
default_git_account = None
if cros_build_lib.HostIsCIBuilder(golo_only=True):
default_git_account = '*****@*****.**'
elif cros_build_lib.HostIsCIBuilder(gce_only=True):
default_git_account = '*****@*****.**'
parser = commandline.ArgumentParser(usage=__doc__, add_help=False)
parser.add_argument('--dryrun',
action='store_true',
default=False,
help="Don't commit changes or send out emails.")
parser.add_argument(
'--user_email',
required=False,
default=default_git_account,
help='Email address to use when comitting changes.')
parser.add_argument('--workdir',
default=os.path.join(os.getcwd(), 'chrome_src'),
help=('Path to a checkout of the chrome src. '
'Defaults to PWD/chrome_src'))
return parser
def _AssertGclientConfigSpec(self, expected_spec, use_cache=True):
if cros_build_lib.HostIsCIBuilder() and use_cache:
if cros_build_lib.IsInsideChroot():
expected_spec += "cache_dir = '/tmp/b/git-cache'\n"
else:
expected_spec += "cache_dir = '/b/git-cache'\n"
self.rc.assertCommandContains(
('gclient', 'config', '--spec', expected_spec), cwd=self._TEST_CWD)
def testBuildBotOnNonCIBuilder(self):
"""Test BuildBot On Non-CIBuilder
Buildbot should quite if run in a non-CIBuilder without
both debug and remote.
"""
if not cros_build_lib.HostIsCIBuilder():
self.assertRaises(cros_build_lib.DieSystemExit, self.assertMain,
['--buildbot', 'x86-generic-paladin'])
def _GetGclientSpec(internal, rev, template, use_cache):
"""Return a formatted gclient spec.
See WriteConfigFile below.
"""
solutions = _GetGclientSolutions(internal=internal,
rev=rev,
template=template)
result = 'solutions = %s\n' % pprint.pformat(solutions)
# Horrible hack, I will go to hell for this. The bots need to have a git
# cache set up; but how can we tell whether this code is running on a bot
# or a developer's machine?
if cros_build_lib.HostIsCIBuilder() and use_cache:
result += "cache_dir = '/b/git-cache'\n"
return result
def testHostIsCIBuilder(self):
"""Test HostIsCIBuilder."""
fq_hostname_golo = 'test.golo.chromium.org'
fq_hostname_gce_1 = 'test.chromeos-bot.internal'
fq_hostname_gce_2 = 'test.chrome.corp.google.com'
fq_hostname_invalid = 'test'
self.assertTrue(cros_build_lib.HostIsCIBuilder(fq_hostname_golo))
self.assertTrue(cros_build_lib.HostIsCIBuilder(fq_hostname_gce_1))
self.assertTrue(cros_build_lib.HostIsCIBuilder(fq_hostname_gce_2))
self.assertFalse(cros_build_lib.HostIsCIBuilder(fq_hostname_invalid))
self.assertFalse(
cros_build_lib.HostIsCIBuilder(fq_hostname=fq_hostname_golo,
gce_only=True))
self.assertFalse(
cros_build_lib.HostIsCIBuilder(fq_hostname=fq_hostname_gce_1,
golo_only=True))
def __init__(self, goma_dir, goma_client_json, goma_tmp_dir=None,
stage_name=None):
"""Initializes Goma instance.
This ensures that |self.goma_log_dir| directory exists (if missing,
creates it).
Args:
goma_dir: Path to the goma directory (outside of chroot).
goma_client_json: Path to the service account json file to use goma.
On bots, this must be specified, otherwise raise a ValueError.
On local, this is optional, and can be set to None.
goma_tmp_dir: Path to the GOMA_TMP_DIR to be passed to goma programs.
If given, it is used. If not given, creates a directory under
/tmp in the chroot, expecting that the directory is removed in the
next run's clean up phase on bots.
stage_name: optional name of the currently running stage. E.g.
"build_packages" or "test_simple_chrome_workflow". If this is set
deps cache is enabled.
Raises:
ValueError if 1) |goma_dir| does not point to a directory, 2)
on bots, but |goma_client_json| is not given, 3) |goma_client_json|
is given, but it does not point to a file, or 4) if |goma_tmp_dir| is
given but it does not point to a directory.
"""
# Sanity checks of given paths.
if not os.path.isdir(goma_dir):
raise ValueError('goma_dir does not point a directory: %s' % (goma_dir,))
# If this script runs on bot, service account json file needs to be
# provided, otherwise it cannot access to goma service.
if cros_build_lib.HostIsCIBuilder() and goma_client_json is None:
raise ValueError(
'goma is enabled on bot, but goma_client_json is not provided')
# If goma_client_json file is provided, it must be an existing file.
if goma_client_json and not os.path.isfile(goma_client_json):
raise ValueError(
'Goma client json file is missing: %s' % (goma_client_json,))
# If goma_tmp_dir is provided, it must be an existing directory.
if goma_tmp_dir and not os.path.isdir(goma_tmp_dir):
raise ValueError(
'GOMA_TMP_DIR does not point a directory: %s' % (goma_tmp_dir,))
self.goma_dir = goma_dir
self.goma_client_json = goma_client_json
if stage_name:
self.goma_cache = os.path.join(goma_dir, 'goma_cache', stage_name)
osutils.SafeMakedirs(self.goma_cache)
else:
self.goma_cache = None
if goma_tmp_dir is None:
# If |goma_tmp_dir| is not given, create GOMA_TMP_DIR (goma
# compiler_proxy's working directory), and its log directory.
# Create unique directory by mkdtemp under chroot's /tmp.
# Expect this directory is removed in next run's clean up phase.
goma_tmp_dir = tempfile.mkdtemp(
prefix='goma_tmp_dir.', dir=path_util.FromChrootPath('/tmp'))
self.goma_tmp_dir = goma_tmp_dir
# Create log directory if not exist.
if not os.path.isdir(self.goma_log_dir):
os.mkdir(self.goma_log_dir)
def main(argv):
# We get false positives with the options object.
# pylint: disable=attribute-defined-outside-init
# Turn on strict sudo checks.
cros_build_lib.STRICT_SUDO = True
# Set umask to 022 so files created by buildbot are readable.
os.umask(0o22)
parser = _CreateParser()
options = ParseCommandLine(parser, argv)
# Fetch our site_config now, because we need it to do anything else.
site_config = config_lib.GetConfig()
_PostParseCheck(parser, options, site_config)
cros_build_lib.AssertOutsideChroot()
if options.enable_buildbot_tags:
logging.EnableBuildbotMarkers()
if (options.buildbot and not options.debug
and not options.build_config_name == constants.BRANCH_UTIL_CONFIG
and not cros_build_lib.HostIsCIBuilder()):
# --buildbot can only be used on a real builder, unless it's debug, or
# 'branch-util'.
cros_build_lib.Die('This host is not a supported build machine.')
# Only one config arg is allowed in this mode, which was confirmed earlier.
build_config = site_config[options.build_config_name]
# TODO: Re-enable this block when reference_repo support handles this
# properly. (see chromium:330775)
# if options.reference_repo is None:
# repo_path = os.path.join(options.sourceroot, '.repo')
# # If we're being run from a repo checkout, reuse the repo's git pool to
# # cut down on sync time.
# if os.path.exists(repo_path):
# options.reference_repo = options.sourceroot
if options.reference_repo:
if not os.path.exists(options.reference_repo):
parser.error('Reference path %s does not exist' %
(options.reference_repo, ))
elif not os.path.exists(os.path.join(options.reference_repo, '.repo')):
parser.error('Reference path %s does not look to be the base of a '
'repo checkout; no .repo exists in the root.' %
(options.reference_repo, ))
if (options.buildbot or options.remote_trybot) and not options.resume:
if not options.cgroups:
parser.error(
'Options --buildbot/--remote-trybot and --nocgroups cannot '
'be used together. Cgroup support is required for '
'buildbot/remote-trybot mode.')
if not cgroups.Cgroup.IsSupported():
parser.error(
'Option --buildbot/--remote-trybot was given, but this '
'system does not support cgroups. Failing.')
missing = osutils.FindMissingBinaries(_BUILDBOT_REQUIRED_BINARIES)
if missing:
parser.error(
'Option --buildbot/--remote-trybot requires the following '
"binaries which couldn't be found in $PATH: %s" %
(', '.join(missing)))
if options.reference_repo:
options.reference_repo = os.path.abspath(options.reference_repo)
# Sanity check of buildroot- specifically that it's not pointing into the
# midst of an existing repo since git-repo doesn't support nesting.
if (not repository.IsARepoRoot(options.buildroot)
and git.FindRepoDir(options.buildroot)):
cros_build_lib.Die(
'Configured buildroot %s is a subdir of an existing repo checkout.'
% options.buildroot)
if not options.log_dir:
options.log_dir = os.path.join(options.buildroot, _DEFAULT_LOG_DIR)
log_file = None
if options.tee:
log_file = os.path.join(options.log_dir, _BUILDBOT_LOG_FILE)
osutils.SafeMakedirs(options.log_dir)
_BackupPreviousLog(log_file)
with cros_build_lib.ContextManagerStack() as stack:
options.preserve_paths = set()
if log_file is not None:
# We don't want the critical section to try to clean up the tee process,
# so we run Tee (forked off) outside of it. This prevents a deadlock
# because the Tee process only exits when its pipe is closed, and the
# critical section accidentally holds on to that file handle.
stack.Add(tee.Tee, log_file)
options.preserve_paths.add(_DEFAULT_LOG_DIR)
critical_section = stack.Add(cleanup.EnforcedCleanupSection)
stack.Add(sudo.SudoKeepAlive)
if not options.resume:
# If we're in resume mode, use our parents tempdir rather than
# nesting another layer.
stack.Add(osutils.TempDir, prefix='cbuildbot-tmp', set_global=True)
logging.debug('Cbuildbot tempdir is %r.', os.environ.get('TMP'))
if options.cgroups:
stack.Add(cgroups.SimpleContainChildren, 'cbuildbot')
# Mark everything between EnforcedCleanupSection and here as having to
# be rolled back via the contextmanager cleanup handlers. This
# ensures that sudo bits cannot outlive cbuildbot, that anything
# cgroups would kill gets killed, etc.
stack.Add(critical_section.ForkWatchdog)
if options.mock_tree_status is not None:
stack.Add(_ObjectMethodPatcher,
tree_status,
'_GetStatus',
return_value=options.mock_tree_status)
if options.mock_slave_status is not None:
with open(options.mock_slave_status, 'r') as f:
mock_statuses = pickle.load(f)
for key, value in mock_statuses.iteritems():
mock_statuses[key] = builder_status_lib.BuilderStatus(
**value)
stack.Add(_ObjectMethodPatcher,
completion_stages.MasterSlaveSyncCompletionStage,
'_FetchSlaveStatuses',
return_value=mock_statuses)
stack.Add(_SetupConnections, options, build_config)
retry_stats.SetupStats()
timeout_display_message = None
# For master-slave builds: Update slave's timeout using master's published
# deadline.
if options.buildbot and options.master_build_id is not None:
slave_timeout = None
if cidb.CIDBConnectionFactory.IsCIDBSetup():
cidb_handle = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder(
)
if cidb_handle:
slave_timeout = cidb_handle.GetTimeToDeadline(
options.master_build_id)
if slave_timeout is not None:
# We artificially set a minimum slave_timeout because '0' is handled
# specially, and because we don't want to timeout while trying to set
# things up.
slave_timeout = max(slave_timeout, 20)
if options.timeout == 0 or slave_timeout < options.timeout:
logging.info(
'Updating slave build timeout to %d seconds enforced '
'by the master', slave_timeout)
options.timeout = slave_timeout
timeout_display_message = (
'This build has reached the timeout deadline set by the master. '
'Either this stage or a previous one took too long (see stage '
'timing historical summary in ReportStage) or the build failed '
'to start on time.')
else:
logging.warning(
'Could not get master deadline for master-slave build. '
'Can not set slave timeout.')
if options.timeout > 0:
stack.Add(timeout_util.FatalTimeout, options.timeout,
timeout_display_message)
try:
_RunBuildStagesWrapper(options, site_config, build_config)
except failures_lib.ExitEarlyException as ex:
# This build finished successfully. Do not re-raise ExitEarlyException.
logging.info('One stage exited early: %s', ex)
def __init__(self,
goma_dir,
goma_client_json,
goma_tmp_dir=None,
stage_name=None,
chromeos_goma_dir=None,
chroot_dir=None,
goma_approach=None):
"""Initializes Goma instance.
This ensures that |self.goma_log_dir| directory exists (if missing,
creates it).
Args:
goma_dir: Path to the Goma client used for simplechrome
(outside of chroot).
goma_client_json: Path to the service account json file to use goma.
On bots, this must be specified, otherwise raise a ValueError.
On local, this is optional, and can be set to None.
goma_tmp_dir: Path to the GOMA_TMP_DIR to be passed to goma programs.
If given, it is used. If not given, creates a directory under
/tmp in the chroot, expecting that the directory is removed in the
next run's clean up phase on bots.
stage_name: optional name of the currently running stage. E.g.
"build_packages" or "test_simple_chrome_workflow". If this is set
deps cache is enabled.
chromeos_goma_dir: Path to the Goma client used for build package.
path should be represented as outside of chroot.
If None, goma_dir will be used instead.
chroot_dir: The base chroot path to use when the chroot path is not at
the default location.
goma_approach: Indicates some extra environment variables to set when
testing alternative goma approaches.
Raises:
ValueError if 1) |goma_dir| does not point to a directory, 2)
on bots, but |goma_client_json| is not given, 3) |goma_client_json|
is given, but it does not point to a file, or 4) if |goma_tmp_dir| is
given but it does not point to a directory.
"""
# Sanity checks of given paths.
if not os.path.isdir(goma_dir):
raise ValueError('goma_dir does not point a directory: %s' %
(goma_dir, ))
# If this script runs on bot, service account json file needs to be
# provided, otherwise it cannot access to goma service.
if cros_build_lib.HostIsCIBuilder() and goma_client_json is None:
raise ValueError(
'goma is enabled on bot, but goma_client_json is not provided')
# If goma_client_json file is provided, it must be an existing file.
if goma_client_json and not os.path.isfile(goma_client_json):
raise ValueError('Goma client json file is missing: %s' %
(goma_client_json, ))
# If goma_tmp_dir is provided, it must be an existing directory.
if goma_tmp_dir and not os.path.isdir(goma_tmp_dir):
raise ValueError('GOMA_TMP_DIR does not point a directory: %s' %
(goma_tmp_dir, ))
self.linux_goma_dir = goma_dir
self.chromeos_goma_dir = chromeos_goma_dir
self.goma_approach = goma_approach
# If Goma dir for ChromeOS SDK does not set, fallback to use goma_dir.
if self.chromeos_goma_dir is None:
self.chromeos_goma_dir = goma_dir
# Sanity checks of given paths.
if not os.path.isdir(self.chromeos_goma_dir):
raise ValueError(
'chromeos_goma_dir does not point a directory: %s' %
(self.chromeos_goma_dir, ))
self.goma_client_json = goma_client_json
if stage_name:
self.goma_cache = os.path.join(goma_dir, 'goma_cache', stage_name)
osutils.SafeMakedirs(self.goma_cache)
else:
self.goma_cache = None
if goma_tmp_dir is None:
# path_util depends on the chroot directory existing at
# SOURCE_ROOT/chroot. This assumption is not valid for Luci builders,
# but generally shouldn't be an assumption anyway since we allow setting
# the chroot location. This block, and the two a few lines down, bypass
# path_util to compensate for those assumptions.
# TODO(crbug.com/1014138) Cleanup when path_util can handle custom chroot.
if chroot_dir:
chroot_tmp = os.path.join(chroot_dir, 'tmp')
else:
chroot_tmp = path_util.FromChrootPath('/tmp')
# If |goma_tmp_dir| is not given, create GOMA_TMP_DIR (goma
# compiler_proxy's working directory), and its log directory.
# Create unique directory by mkdtemp under chroot's /tmp.
# Expect this directory is removed in next run's clean up phase.
goma_tmp_dir = tempfile.mkdtemp(prefix='goma_tmp_dir.',
dir=chroot_tmp)
self.goma_tmp_dir = goma_tmp_dir
if chroot_dir:
self.chroot_goma_tmp_dir = os.path.join(
'/', os.path.relpath(self.goma_tmp_dir, chroot_dir))
else:
self.chroot_goma_tmp_dir = path_util.ToChrootPath(
self.goma_tmp_dir)
# Create log directory if not exist.
if not os.path.isdir(self.goma_log_dir):
os.mkdir(self.goma_log_dir)
if chroot_dir:
self.chroot_goma_log_dir = os.path.join(
'/', os.path.relpath(self.goma_log_dir, chroot_dir))
else:
self.chroot_goma_log_dir = path_util.ToChrootPath(
self.goma_log_dir)
