def __init__(self,
name,
source_path,
chromeos_root,
board,
incremental,
build_env,
gcc_enable_ccache=False):
self._name = name
self._source_path = misc.CanonicalizePath(source_path)
self._chromeos_root = chromeos_root
self._board = board
self._ctarget = misc.GetCtargetFromBoard(self._board,
self._chromeos_root)
self._gcc_libs_dest = misc.GetGccLibsDestForBoard(
self._board, self._chromeos_root)
self.tag = '%s-%s' % (name, self._ctarget)
self._ce = command_executer.GetCommandExecuter()
self._mask_file = os.path.join(
self._chromeos_root, 'chroot',
'etc/portage/package.mask/cross-%s' % self._ctarget)
self._new_mask_file = None
self._chroot_source_path = os.path.join(
constants.MOUNTED_TOOLCHAIN_ROOT, self._name).lstrip('/')
self._incremental = incremental
self._build_env = build_env
self._gcc_enable_ccache = gcc_enable_ccache
def GetXbuddyPath(self, path_str, autotest_path, board, chromeos_root,
log_level):
prefix = 'remote'
l = logger.GetLogger()
if (path_str.find('trybot') < 0 and path_str.find('toolchain') < 0
and path_str.find(board) < 0):
xbuddy_path = '%s/%s/%s' % (prefix, board, path_str)
else:
xbuddy_path = '%s/%s' % (prefix, path_str)
image_downloader = ImageDownloader(l, log_level)
image_and_autotest_path = image_downloader.Run(
misc.CanonicalizePath(chromeos_root), xbuddy_path, autotest_path)
return image_and_autotest_path
def UploadGccPatch(chromeos_root, gcc_dir, branch):
"""Upload local gcc to gerrit and get the CL number."""
ce = command_executer.GetCommandExecuter()
gcc_dir = misc.CanonicalizePath(gcc_dir)
gcc_path = os.path.join(chromeos_root, 'src/third_party/gcc')
assert os.path.isdir(gcc_path), ('{0} is not a valid chromeos root'
.format(chromeos_root))
assert os.path.isdir(gcc_dir), ('{0} is not a valid dir for gcc'
'source'.format(gcc_dir))
os.chdir(gcc_path)
RemoveOldBranch()
if not branch:
branch = 'master'
branch = GetGccBranch(branch)
command = ('git checkout -b {0} -t {1} && ' 'rm -rf *'.format(BRANCH, branch))
ce.RunCommand(command, print_to_console=False)
command = ("rsync -az --exclude='*.svn' --exclude='*.git'"
' {0}/ .'.format(gcc_dir))
ce.RunCommand(command)
return UploadPatch(gcc_dir)
def UploadManifest(manifest, chromeos_root, branch='master'):
"""Copy the manifest to $chromeos_root/manifest-internal and upload."""
chromeos_root = misc.CanonicalizePath(chromeos_root)
manifest_dir = os.path.join(chromeos_root, 'manifest-internal')
os.chdir(manifest_dir)
ce = command_executer.GetCommandExecuter()
RemoveOldBranch()
if branch != 'master':
branch = '{0}'.format(branch)
command = 'git checkout -b {0} -t cros-internal/{1}'.format(BRANCH, branch)
ret = ce.RunCommand(command)
if ret:
raise RuntimeError('Command {0} failed'.format(command))
# We remove the default.xml, which is the symbolic link of full.xml.
# After that, we copy our xml file to default.xml.
# We did this because the full.xml might be updated during the
# run of the script.
os.remove(os.path.join(manifest_dir, 'default.xml'))
shutil.copyfile(manifest, os.path.join(manifest_dir, 'default.xml'))
return UploadPatch(manifest)
def __init__(self,
name,
chromeos_image,
autotest_path,
chromeos_root,
board,
remote,
image_args,
cache_dir,
cache_only,
log_level,
compiler,
chrome_src=None):
self.image_type = self._GetImageType(chromeos_image)
# Expand ~
chromeos_root = os.path.expanduser(chromeos_root)
if self.image_type == 'local':
chromeos_image = os.path.expanduser(chromeos_image)
self.name = name
self.chromeos_image = chromeos_image
self.autotest_path = autotest_path
self.board = board
self.remote = remote
self.image_args = image_args
self.cache_dir = cache_dir
self.cache_only = cache_only
self.log_level = log_level
self.chrome_version = ''
self.compiler = compiler
if not chromeos_root:
if self.image_type == 'local':
chromeos_root = FileUtils().ChromeOSRootFromImage(
chromeos_image)
if not chromeos_root:
raise RuntimeError(
"No ChromeOS root given for label '%s' and could "
"not determine one from image path: '%s'." %
(name, chromeos_image))
else:
chromeos_root = FileUtils().CanonicalizeChromeOSRoot(chromeos_root)
if not chromeos_root:
raise RuntimeError(
"Invalid ChromeOS root given for label '%s': '%s'." %
(name, chromeos_root))
self.chromeos_root = chromeos_root
if not chrome_src:
self.chrome_src = os.path.join(
self.chromeos_root,
'.cache/distfiles/target/chrome-src-internal')
if not os.path.exists(self.chrome_src):
self.chrome_src = os.path.join(
self.chromeos_root, '.cache/distfiles/target/chrome-src')
else:
chromeos_src = misc.CanonicalizePath(chrome_src)
if not chromeos_src:
raise RuntimeError(
"Invalid Chrome src given for label '%s': '%s'." %
(name, chrome_src))
self.chrome_src = chromeos_src
self._SetupChecksum()
def Main(argv):
"""Build Chrome browser."""
cmd_executer = command_executer.GetCommandExecuter()
parser = argparse.ArgumentParser()
parser.add_argument('--chromeos_root',
dest='chromeos_root',
help='Target directory for ChromeOS installation.')
parser.add_argument('--version', dest='version')
parser.add_argument('--clean',
dest='clean',
default=False,
action='store_true',
help=('Clean the /var/cache/chromeos-chrome/'
'chrome-src/src/out_$board dir'))
parser.add_argument('--env',
dest='env',
default='',
help='Use the following env')
parser.add_argument('--ebuild_version',
dest='ebuild_version',
help='Use this ebuild instead of the default one.')
parser.add_argument('--cflags',
dest='cflags',
default='',
help='CFLAGS for the ChromeOS packages')
parser.add_argument('--cxxflags',
dest='cxxflags',
default='',
help='CXXFLAGS for the ChromeOS packages')
parser.add_argument('--ldflags',
dest='ldflags',
default='',
help='LDFLAGS for the ChromeOS packages')
parser.add_argument('--board',
dest='board',
help='ChromeOS target board, e.g. x86-generic')
parser.add_argument('--no_build_image',
dest='no_build_image',
default=False,
action='store_true',
help=('Skip build image after building browser.'
'Defaults to False.'))
parser.add_argument('--label',
dest='label',
help='Optional label to apply to the ChromeOS image.')
parser.add_argument('--build_image_args',
default='',
dest='build_image_args',
help='Optional arguments to build_image.')
parser.add_argument('--cros_workon',
dest='cros_workon',
help='Build using external source tree.')
parser.add_argument('--dev',
dest='dev',
default=False,
action='store_true',
help=('Build a dev (eg. writable/large) image. '
'Defaults to False.'))
parser.add_argument('--debug',
dest='debug',
default=False,
action='store_true',
help=('Build chrome browser using debug mode. '
'This option implies --dev. Defaults to false.'))
parser.add_argument('--verbose',
dest='verbose',
default=False,
action='store_true',
help='Build with verbose information.')
options = parser.parse_args(argv)
if options.chromeos_root is None:
Usage(parser, '--chromeos_root must be set')
if options.board is None:
Usage(parser, '--board must be set')
if options.version is None:
logger.GetLogger().LogOutput('No Chrome version given so '
'using the default checked in version.')
chrome_version = ''
else:
chrome_version = 'CHROME_VERSION=%s' % options.version
if options.dev and options.no_build_image:
logger.GetLogger().LogOutput(
"\"--dev\" is meaningless if \"--no_build_image\" is given.")
if options.debug:
options.dev = True
options.chromeos_root = misc.CanonicalizePath(options.chromeos_root)
unmask_env = 'ACCEPT_KEYWORDS=~*'
if options.ebuild_version:
ebuild_version = '=%s' % options.ebuild_version
options.env = '%s %s' % (options.env, unmask_env)
else:
ebuild_version = 'chromeos-chrome'
if options.cros_workon and not (os.path.isdir(
options.cros_workon) and os.path.exists(
os.path.join(options.cros_workon, 'src/chromeos/BUILD.gn'))):
Usage(parser,
'--cros_workon must be a valid chromium browser checkout.')
if options.verbose:
options.env = misc.MergeEnvStringWithDict(
options.env, {'USE': 'chrome_internal verbose'})
else:
options.env = misc.MergeEnvStringWithDict(options.env,
{'USE': 'chrome_internal'})
if options.debug:
options.env = misc.MergeEnvStringWithDict(options.env,
{'BUILDTYPE': 'Debug'})
if options.clean:
misc.RemoveChromeBrowserObjectFiles(options.chromeos_root,
options.board)
chrome_origin = 'SERVER_SOURCE'
if options.cros_workon:
chrome_origin = 'LOCAL_SOURCE'
command = 'cros_workon --board={0} start chromeos-chrome'.format(
options.board)
ret = cmd_executer.ChrootRunCommandWOutput(options.chromeos_root,
command)
# cros_workon start returns non-zero if chromeos-chrome is already a
# cros_workon package.
if ret[0] and ret[2].find(
'WARNING : Already working on chromeos-base/chromeos-chrome'
) == -1:
logger.GetLogger().LogFatal('cros_workon chromeos-chrome failed.')
# Return value is non-zero means we do find the "Already working on..."
# message, keep the information, so later on we do not revert the
# cros_workon status.
cros_workon_keep = (ret[0] != 0)
# Emerge the browser
emerge_browser_command = ('CHROME_ORIGIN={0} {1} '
'CFLAGS="$(portageq-{2} envvar CFLAGS) {3}" '
'LDFLAGS="$(portageq-{2} envvar LDFLAGS) {4}" '
'CXXFLAGS="$(portageq-{2} envvar CXXFLAGS) {5}" '
'{6} emerge-{2} --buildpkg {7}').format(
chrome_origin, chrome_version, options.board,
options.cflags, options.ldflags,
options.cxxflags, options.env,
ebuild_version)
cros_sdk_options = ''
if options.cros_workon:
cros_sdk_options = '--chrome_root={0}'.format(options.cros_workon)
ret = cmd_executer.ChrootRunCommand(options.chromeos_root,
emerge_browser_command,
cros_sdk_options=cros_sdk_options)
logger.GetLogger().LogFatalIf(ret, 'build_packages failed')
if options.cros_workon and not cros_workon_keep:
command = 'cros_workon --board={0} stop chromeos-chrome'.format(
options.board)
ret = cmd_executer.ChrootRunCommand(options.chromeos_root, command)
# cros_workon failed, not a fatal one, just report it.
if ret:
print('cros_workon stop chromeos-chrome failed.')
if options.no_build_image:
return ret
# Finally build the image
ret = cmd_executer.ChrootRunCommand(
options.chromeos_root, '{0} {1} {2} {3}'.format(
unmask_env, options.env,
misc.GetBuildImageCommand(options.board, dev=options.dev),
options.build_image_args))
logger.GetLogger().LogFatalIf(ret, 'build_image failed')
flags_file_name = 'chrome_flags.txt'
flags_file_path = '{0}/src/build/images/{1}/latest/{2}'.format(
options.chromeos_root, options.board, flags_file_name)
flags_file = open(flags_file_path, 'wb')
flags_file.write('CFLAGS={0}\n'.format(options.cflags))
flags_file.write('CXXFLAGS={0}\n'.format(options.cxxflags))
flags_file.write('LDFLAGS={0}\n'.format(options.ldflags))
flags_file.close()
if options.label:
image_dir_path = '{0}/src/build/images/{1}/latest'.format(
options.chromeos_root, options.board)
real_image_dir_path = os.path.realpath(image_dir_path)
command = 'ln -sf -T {0} {1}/{2}'.format(
os.path.basename(real_image_dir_path),\
os.path.dirname(real_image_dir_path),\
options.label)
ret = cmd_executer.RunCommand(command)
logger.GetLogger().LogFatalIf(
ret, 'Failed to apply symlink label %s' % options.label)
return ret
def Main(argv):
"""The main function."""
# Common initializations
parser = argparse.ArgumentParser()
parser.add_argument('-c',
'--chromeos_root',
required=True,
dest='chromeos_root',
help='The chromeos_root')
parser.add_argument('-g',
'--gcc_dir',
default='',
dest='gcc_dir',
help='The gcc dir')
parser.add_argument('-t',
'--target',
required=True,
dest='target',
help=('The target to be build, the list is at'
' $(chromeos_root)/chromite/buildbot/cbuildbot'
' --list -all'))
parser.add_argument('-l', '--local', action='store_true')
parser.add_argument('-d',
'--dest_dir',
dest='dest_dir',
help=('The dir to build the whole chromeos if'
' --local is set'))
parser.add_argument('--chrome_version',
dest='chrome_version',
default='',
help='The chrome version to use. '
'Default it will use the latest one.')
parser.add_argument('--chromeos_version',
dest='chromeos_version',
default='',
help=('The chromeos version to use.'
'(1) A release version in the format: '
"'\d+\.\d+\.\d+\.\d+.*'"
"(2) 'latest_lkgm' for the latest lkgm version"))
parser.add_argument('-r',
'--replace_sysroot',
action='store_true',
help=('Whether or not to replace the build/$board dir'
'under the chroot of chromeos_root and copy '
'the image to src/build/image/$board/latest.'
' Default is False'))
parser.add_argument('-b',
'--branch',
dest='branch',
default='',
help=('The branch to run trybot, default is None'))
parser.add_argument('-p',
'--patch',
dest='patch',
default='',
help=('The patches to be applied, the patches numbers '
"be seperated by ','"))
script_dir = os.path.dirname(os.path.realpath(__file__))
args = parser.parse_args(argv[1:])
target = args.target
if args.patch:
patch = args.patch.split(',')
else:
patch = []
chromeos_root = misc.CanonicalizePath(args.chromeos_root)
if args.chromeos_version and args.branch:
raise RuntimeError(
'You can not set chromeos_version and branch at the '
'same time.')
manifests = None
if args.branch:
chromeos_version = ''
branch = args.branch
else:
chromeos_version = args.chromeos_version
manifests = manifest_versions.ManifestVersions()
if chromeos_version == 'latest_lkgm':
chromeos_version = manifests.TimeToVersion(
time.mktime(time.gmtime()))
logger.GetLogger().LogOutput('found version %s for latest LKGM' %
(chromeos_version))
# TODO: this script currently does not handle the case where the version
# is not in the "master" branch
branch = 'master'
if chromeos_version:
manifest_patch = GetManifestPatch(manifests, chromeos_version,
chromeos_root)
patch.append(manifest_patch)
if args.gcc_dir:
# TODO: everytime we invoke this script we are getting a different
# patch for GCC even if GCC has not changed. The description should
# be based on the MD5 of the GCC patch contents.
patch.append(UploadGccPatch(chromeos_root, args.gcc_dir, branch))
description = RunRemote(chromeos_root, branch, patch, args.local, target,
args.chrome_version, args.dest_dir)
if args.local or not args.dest_dir:
# TODO: We are not checktng the result of cbuild_bot in here!
return 0
# return value:
# 0 => build bot was successful and image was put where requested
# 1 => Build bot FAILED but image was put where requested
# 2 => Build bot failed or BUild bot was successful but and image was
# not generated or could not be put where expected
os.chdir(script_dir)
dest_dir = misc.CanonicalizePath(args.dest_dir)
(bot_result, build_id) = FindBuildId(description)
if bot_result > 0 and build_id > 0:
logger.GetLogger().LogError(
'Remote trybot failed but image was generated')
bot_result = 1
elif bot_result > 0:
logger.GetLogger().LogError(
'Remote trybot failed. No image was generated')
return 2
if 'toolchain' in branch:
chromeos_version = FindVersionForToolchain(branch, chromeos_root)
assert not manifest_versions.IsRFormatCrosVersion(chromeos_version)
DownloadImage(target, build_id, dest_dir, chromeos_version)
ret = UnpackImage(dest_dir)
if ret != 0:
return 2
# todo: return a more inteligent return value
if not args.replace_sysroot:
return bot_result
ret = ReplaceSysroot(chromeos_root, args.dest_dir, target)
if ret != 0:
return 2
# got an image and we were successful in placing it where requested
return bot_result
def __init__(self, name, remote, working_directory, chromeos_root,
cache_conditions, labels, benchmarks, experiment_file, email_to,
acquire_timeout, log_dir, log_level, share_cache,
results_directory, locks_directory):
self.name = name
self.working_directory = working_directory
self.remote = remote
self.chromeos_root = chromeos_root
self.cache_conditions = cache_conditions
self.experiment_file = experiment_file
self.email_to = email_to
if not results_directory:
self.results_directory = os.path.join(self.working_directory,
self.name + '_results')
else:
self.results_directory = misc.CanonicalizePath(results_directory)
self.log_dir = log_dir
self.log_level = log_level
self.labels = labels
self.benchmarks = benchmarks
self.num_complete = 0
self.num_run_complete = 0
self.share_cache = share_cache
self.active_threads = []
# If locks_directory (self.lock_dir) not blank, we will use the file
# locking mechanism; if it is blank then we will use the AFE server
# locking mechanism.
self.locks_dir = locks_directory
self.locked_machines = []
if not remote:
raise RuntimeError('No remote hosts specified')
if not self.benchmarks:
raise RuntimeError('No benchmarks specified')
if not self.labels:
raise RuntimeError('No labels specified')
# We need one chromeos_root to run the benchmarks in, but it doesn't
# matter where it is, unless the ABIs are different.
if not chromeos_root:
for label in self.labels:
if label.chromeos_root:
chromeos_root = label.chromeos_root
break
if not chromeos_root:
raise RuntimeError('No chromeos_root given and could not determine '
'one from the image path.')
machine_manager_fn = MachineManager
if test_flag.GetTestMode():
machine_manager_fn = MockMachineManager
self.machine_manager = machine_manager_fn(chromeos_root, acquire_timeout,
log_level, locks_directory)
self.l = logger.GetLogger(log_dir)
for machine in self.remote:
# machine_manager.AddMachine only adds reachable machines.
self.machine_manager.AddMachine(machine)
# Now machine_manager._all_machines contains a list of reachable
# machines. This is a subset of self.remote. We make both lists the same.
self.remote = [m.name for m in self.machine_manager.GetAllMachines()]
if not self.remote:
raise RuntimeError('No machine available for running experiment.')
for label in labels:
# We filter out label remotes that are not reachable (not in
# self.remote). So each label.remote is a sublist of experiment.remote.
label.remote = [r for r in label.remote if r in self.remote]
try:
self.machine_manager.ComputeCommonCheckSum(label)
except BadChecksum:
# Force same image on all machines, then we do checksum again. No
# bailout if checksums still do not match.
self.machine_manager.ForceSameImageToAllMachines(label)
self.machine_manager.ComputeCommonCheckSum(label)
self.machine_manager.ComputeCommonCheckSumString(label)
self.start_time = None
self.benchmark_runs = self._GenerateBenchmarkRuns()
self._schedv2 = None
self._internal_counter_lock = Lock()
def Main(argv):
"""The main function."""
# Common initializations
parser = argparse.ArgumentParser()
parser.add_argument('-c',
'--chromeos_root',
dest='chromeos_root',
default='../../',
help=('ChromeOS root checkout directory'
' uses ../.. if none given.'))
parser.add_argument('-g',
'--gcc_dir',
dest='gcc_dir',
help='The directory where gcc resides.')
parser.add_argument('--binutils_dir',
dest='binutils_dir',
help='The directory where binutils resides.')
parser.add_argument('-x',
'--gdb_dir',
dest='gdb_dir',
help='The directory where gdb resides.')
parser.add_argument('-b',
'--board',
dest='board',
default='x86-alex',
help='The target board.')
parser.add_argument('-n',
'--noincremental',
dest='noincremental',
default=False,
action='store_true',
help='Use FEATURES=keepwork to do incremental builds.')
parser.add_argument('--cflags',
dest='cflags',
default='',
help='Build a compiler with specified CFLAGS')
parser.add_argument('--cxxflags',
dest='cxxflags',
default='',
help='Build a compiler with specified CXXFLAGS')
parser.add_argument('--cflags_for_target',
dest='cflags_for_target',
default='',
help='Build the target libraries with specified flags')
parser.add_argument('--cxxflags_for_target',
dest='cxxflags_for_target',
default='',
help='Build the target libraries with specified flags')
parser.add_argument('--ldflags',
dest='ldflags',
default='',
help='Build a compiler with specified LDFLAGS')
parser.add_argument('-d',
'--debug',
dest='debug',
default=False,
action='store_true',
help='Build a compiler with -g3 -O0 appended to both'
' CFLAGS and CXXFLAGS.')
parser.add_argument('-m',
'--mount_only',
dest='mount_only',
default=False,
action='store_true',
help='Just mount the tool directories.')
parser.add_argument('-u',
'--unmount_only',
dest='unmount_only',
default=False,
action='store_true',
help='Just unmount the tool directories.')
parser.add_argument(
'--extra_use_flags',
dest='extra_use_flags',
default='',
help='Extra flag for USE, to be passed to the ebuild. '
"('multislot' and 'mounted_<tool>' are always passed.)")
parser.add_argument('--gcc_enable_ccache',
dest='gcc_enable_ccache',
default=False,
action='store_true',
help='Enable ccache for the gcc invocations')
options = parser.parse_args(argv)
chromeos_root = misc.CanonicalizePath(options.chromeos_root)
if options.gcc_dir:
gcc_dir = misc.CanonicalizePath(options.gcc_dir)
assert gcc_dir and os.path.isdir(gcc_dir), 'gcc_dir does not exist!'
if options.binutils_dir:
binutils_dir = misc.CanonicalizePath(options.binutils_dir)
assert os.path.isdir(binutils_dir), 'binutils_dir does not exist!'
if options.gdb_dir:
gdb_dir = misc.CanonicalizePath(options.gdb_dir)
assert os.path.isdir(gdb_dir), 'gdb_dir does not exist!'
if options.unmount_only:
options.mount_only = False
elif options.mount_only:
options.unmount_only = False
build_env = {}
if options.cflags:
build_env['CFLAGS'] = '`portageq envvar CFLAGS` ' + options.cflags
if options.cxxflags:
build_env[
'CXXFLAGS'] = '`portageq envvar CXXFLAGS` ' + options.cxxflags
if options.cflags_for_target:
build_env['CFLAGS_FOR_TARGET'] = options.cflags_for_target
if options.cxxflags_for_target:
build_env['CXXFLAGS_FOR_TARGET'] = options.cxxflags_for_target
if options.ldflags:
build_env['LDFLAGS'] = options.ldflags
if options.debug:
debug_flags = '-g3 -O0'
if 'CFLAGS' in build_env:
build_env['CFLAGS'] += ' %s' % (debug_flags)
else:
build_env['CFLAGS'] = debug_flags
if 'CXXFLAGS' in build_env:
build_env['CXXFLAGS'] += ' %s' % (debug_flags)
else:
build_env['CXXFLAGS'] = debug_flags
if options.extra_use_flags:
build_env['USE'] = options.extra_use_flags
# Create toolchain parts
toolchain_parts = {}
for board in options.board.split(','):
if options.gcc_dir:
tp = ToolchainPart('gcc', gcc_dir, chromeos_root, board,
not options.noincremental, build_env,
options.gcc_enable_ccache)
toolchain_parts[tp.tag] = tp
tp.RunSetupBoardIfNecessary()
if options.binutils_dir:
tp = ToolchainPart('binutils', binutils_dir, chromeos_root, board,
not options.noincremental, build_env)
toolchain_parts[tp.tag] = tp
tp.RunSetupBoardIfNecessary()
if options.gdb_dir:
tp = ToolchainPart('gdb', gdb_dir, chromeos_root, board,
not options.noincremental, build_env)
toolchain_parts[tp.tag] = tp
tp.RunSetupBoardIfNecessary()
rv = 0
try:
for tag in toolchain_parts:
tp = toolchain_parts[tag]
if options.mount_only or options.unmount_only:
tp.MountSources(options.unmount_only)
else:
rv = rv + tp.Build()
finally:
print('Exiting...')
return rv
