def __init__(self, name, source_path, chromeos_root, board, incremental, build_env, gcc_enable_ccache=False): self._name = name self._source_path = misc.CanonicalizePath(source_path) self._chromeos_root = chromeos_root self._board = board self._ctarget = misc.GetCtargetFromBoard(self._board, self._chromeos_root) self._gcc_libs_dest = misc.GetGccLibsDestForBoard( self._board, self._chromeos_root) self.tag = '%s-%s' % (name, self._ctarget) self._ce = command_executer.GetCommandExecuter() self._mask_file = os.path.join( self._chromeos_root, 'chroot', 'etc/portage/package.mask/cross-%s' % self._ctarget) self._new_mask_file = None self._chroot_source_path = os.path.join( constants.MOUNTED_TOOLCHAIN_ROOT, self._name).lstrip('/') self._incremental = incremental self._build_env = build_env self._gcc_enable_ccache = gcc_enable_ccache
def GetXbuddyPath(self, path_str, autotest_path, board, chromeos_root, log_level): prefix = 'remote' l = logger.GetLogger() if (path_str.find('trybot') < 0 and path_str.find('toolchain') < 0 and path_str.find(board) < 0): xbuddy_path = '%s/%s/%s' % (prefix, board, path_str) else: xbuddy_path = '%s/%s' % (prefix, path_str) image_downloader = ImageDownloader(l, log_level) image_and_autotest_path = image_downloader.Run( misc.CanonicalizePath(chromeos_root), xbuddy_path, autotest_path) return image_and_autotest_path
def UploadGccPatch(chromeos_root, gcc_dir, branch): """Upload local gcc to gerrit and get the CL number.""" ce = command_executer.GetCommandExecuter() gcc_dir = misc.CanonicalizePath(gcc_dir) gcc_path = os.path.join(chromeos_root, 'src/third_party/gcc') assert os.path.isdir(gcc_path), ('{0} is not a valid chromeos root' .format(chromeos_root)) assert os.path.isdir(gcc_dir), ('{0} is not a valid dir for gcc' 'source'.format(gcc_dir)) os.chdir(gcc_path) RemoveOldBranch() if not branch: branch = 'master' branch = GetGccBranch(branch) command = ('git checkout -b {0} -t {1} && ' 'rm -rf *'.format(BRANCH, branch)) ce.RunCommand(command, print_to_console=False) command = ("rsync -az --exclude='*.svn' --exclude='*.git'" ' {0}/ .'.format(gcc_dir)) ce.RunCommand(command) return UploadPatch(gcc_dir)
def UploadManifest(manifest, chromeos_root, branch='master'): """Copy the manifest to $chromeos_root/manifest-internal and upload.""" chromeos_root = misc.CanonicalizePath(chromeos_root) manifest_dir = os.path.join(chromeos_root, 'manifest-internal') os.chdir(manifest_dir) ce = command_executer.GetCommandExecuter() RemoveOldBranch() if branch != 'master': branch = '{0}'.format(branch) command = 'git checkout -b {0} -t cros-internal/{1}'.format(BRANCH, branch) ret = ce.RunCommand(command) if ret: raise RuntimeError('Command {0} failed'.format(command)) # We remove the default.xml, which is the symbolic link of full.xml. # After that, we copy our xml file to default.xml. # We did this because the full.xml might be updated during the # run of the script. os.remove(os.path.join(manifest_dir, 'default.xml')) shutil.copyfile(manifest, os.path.join(manifest_dir, 'default.xml')) return UploadPatch(manifest)
def __init__(self, name, chromeos_image, autotest_path, chromeos_root, board, remote, image_args, cache_dir, cache_only, log_level, compiler, chrome_src=None): self.image_type = self._GetImageType(chromeos_image) # Expand ~ chromeos_root = os.path.expanduser(chromeos_root) if self.image_type == 'local': chromeos_image = os.path.expanduser(chromeos_image) self.name = name self.chromeos_image = chromeos_image self.autotest_path = autotest_path self.board = board self.remote = remote self.image_args = image_args self.cache_dir = cache_dir self.cache_only = cache_only self.log_level = log_level self.chrome_version = '' self.compiler = compiler if not chromeos_root: if self.image_type == 'local': chromeos_root = FileUtils().ChromeOSRootFromImage( chromeos_image) if not chromeos_root: raise RuntimeError( "No ChromeOS root given for label '%s' and could " "not determine one from image path: '%s'." % (name, chromeos_image)) else: chromeos_root = FileUtils().CanonicalizeChromeOSRoot(chromeos_root) if not chromeos_root: raise RuntimeError( "Invalid ChromeOS root given for label '%s': '%s'." % (name, chromeos_root)) self.chromeos_root = chromeos_root if not chrome_src: self.chrome_src = os.path.join( self.chromeos_root, '.cache/distfiles/target/chrome-src-internal') if not os.path.exists(self.chrome_src): self.chrome_src = os.path.join( self.chromeos_root, '.cache/distfiles/target/chrome-src') else: chromeos_src = misc.CanonicalizePath(chrome_src) if not chromeos_src: raise RuntimeError( "Invalid Chrome src given for label '%s': '%s'." % (name, chrome_src)) self.chrome_src = chromeos_src self._SetupChecksum()
def Main(argv): """Build Chrome browser.""" cmd_executer = command_executer.GetCommandExecuter() parser = argparse.ArgumentParser() parser.add_argument('--chromeos_root', dest='chromeos_root', help='Target directory for ChromeOS installation.') parser.add_argument('--version', dest='version') parser.add_argument('--clean', dest='clean', default=False, action='store_true', help=('Clean the /var/cache/chromeos-chrome/' 'chrome-src/src/out_$board dir')) parser.add_argument('--env', dest='env', default='', help='Use the following env') parser.add_argument('--ebuild_version', dest='ebuild_version', help='Use this ebuild instead of the default one.') parser.add_argument('--cflags', dest='cflags', default='', help='CFLAGS for the ChromeOS packages') parser.add_argument('--cxxflags', dest='cxxflags', default='', help='CXXFLAGS for the ChromeOS packages') parser.add_argument('--ldflags', dest='ldflags', default='', help='LDFLAGS for the ChromeOS packages') parser.add_argument('--board', dest='board', help='ChromeOS target board, e.g. x86-generic') parser.add_argument('--no_build_image', dest='no_build_image', default=False, action='store_true', help=('Skip build image after building browser.' 'Defaults to False.')) parser.add_argument('--label', dest='label', help='Optional label to apply to the ChromeOS image.') parser.add_argument('--build_image_args', default='', dest='build_image_args', help='Optional arguments to build_image.') parser.add_argument('--cros_workon', dest='cros_workon', help='Build using external source tree.') parser.add_argument('--dev', dest='dev', default=False, action='store_true', help=('Build a dev (eg. writable/large) image. ' 'Defaults to False.')) parser.add_argument('--debug', dest='debug', default=False, action='store_true', help=('Build chrome browser using debug mode. ' 'This option implies --dev. Defaults to false.')) parser.add_argument('--verbose', dest='verbose', default=False, action='store_true', help='Build with verbose information.') options = parser.parse_args(argv) if options.chromeos_root is None: Usage(parser, '--chromeos_root must be set') if options.board is None: Usage(parser, '--board must be set') if options.version is None: logger.GetLogger().LogOutput('No Chrome version given so ' 'using the default checked in version.') chrome_version = '' else: chrome_version = 'CHROME_VERSION=%s' % options.version if options.dev and options.no_build_image: logger.GetLogger().LogOutput( "\"--dev\" is meaningless if \"--no_build_image\" is given.") if options.debug: options.dev = True options.chromeos_root = misc.CanonicalizePath(options.chromeos_root) unmask_env = 'ACCEPT_KEYWORDS=~*' if options.ebuild_version: ebuild_version = '=%s' % options.ebuild_version options.env = '%s %s' % (options.env, unmask_env) else: ebuild_version = 'chromeos-chrome' if options.cros_workon and not (os.path.isdir( options.cros_workon) and os.path.exists( os.path.join(options.cros_workon, 'src/chromeos/BUILD.gn'))): Usage(parser, '--cros_workon must be a valid chromium browser checkout.') if options.verbose: options.env = misc.MergeEnvStringWithDict( options.env, {'USE': 'chrome_internal verbose'}) else: options.env = misc.MergeEnvStringWithDict(options.env, {'USE': 'chrome_internal'}) if options.debug: options.env = misc.MergeEnvStringWithDict(options.env, {'BUILDTYPE': 'Debug'}) if options.clean: misc.RemoveChromeBrowserObjectFiles(options.chromeos_root, options.board) chrome_origin = 'SERVER_SOURCE' if options.cros_workon: chrome_origin = 'LOCAL_SOURCE' command = 'cros_workon --board={0} start chromeos-chrome'.format( options.board) ret = cmd_executer.ChrootRunCommandWOutput(options.chromeos_root, command) # cros_workon start returns non-zero if chromeos-chrome is already a # cros_workon package. if ret[0] and ret[2].find( 'WARNING : Already working on chromeos-base/chromeos-chrome' ) == -1: logger.GetLogger().LogFatal('cros_workon chromeos-chrome failed.') # Return value is non-zero means we do find the "Already working on..." # message, keep the information, so later on we do not revert the # cros_workon status. cros_workon_keep = (ret[0] != 0) # Emerge the browser emerge_browser_command = ('CHROME_ORIGIN={0} {1} ' 'CFLAGS="$(portageq-{2} envvar CFLAGS) {3}" ' 'LDFLAGS="$(portageq-{2} envvar LDFLAGS) {4}" ' 'CXXFLAGS="$(portageq-{2} envvar CXXFLAGS) {5}" ' '{6} emerge-{2} --buildpkg {7}').format( chrome_origin, chrome_version, options.board, options.cflags, options.ldflags, options.cxxflags, options.env, ebuild_version) cros_sdk_options = '' if options.cros_workon: cros_sdk_options = '--chrome_root={0}'.format(options.cros_workon) ret = cmd_executer.ChrootRunCommand(options.chromeos_root, emerge_browser_command, cros_sdk_options=cros_sdk_options) logger.GetLogger().LogFatalIf(ret, 'build_packages failed') if options.cros_workon and not cros_workon_keep: command = 'cros_workon --board={0} stop chromeos-chrome'.format( options.board) ret = cmd_executer.ChrootRunCommand(options.chromeos_root, command) # cros_workon failed, not a fatal one, just report it. if ret: print('cros_workon stop chromeos-chrome failed.') if options.no_build_image: return ret # Finally build the image ret = cmd_executer.ChrootRunCommand( options.chromeos_root, '{0} {1} {2} {3}'.format( unmask_env, options.env, misc.GetBuildImageCommand(options.board, dev=options.dev), options.build_image_args)) logger.GetLogger().LogFatalIf(ret, 'build_image failed') flags_file_name = 'chrome_flags.txt' flags_file_path = '{0}/src/build/images/{1}/latest/{2}'.format( options.chromeos_root, options.board, flags_file_name) flags_file = open(flags_file_path, 'wb') flags_file.write('CFLAGS={0}\n'.format(options.cflags)) flags_file.write('CXXFLAGS={0}\n'.format(options.cxxflags)) flags_file.write('LDFLAGS={0}\n'.format(options.ldflags)) flags_file.close() if options.label: image_dir_path = '{0}/src/build/images/{1}/latest'.format( options.chromeos_root, options.board) real_image_dir_path = os.path.realpath(image_dir_path) command = 'ln -sf -T {0} {1}/{2}'.format( os.path.basename(real_image_dir_path),\ os.path.dirname(real_image_dir_path),\ options.label) ret = cmd_executer.RunCommand(command) logger.GetLogger().LogFatalIf( ret, 'Failed to apply symlink label %s' % options.label) return ret
def Main(argv): """The main function.""" # Common initializations parser = argparse.ArgumentParser() parser.add_argument('-c', '--chromeos_root', required=True, dest='chromeos_root', help='The chromeos_root') parser.add_argument('-g', '--gcc_dir', default='', dest='gcc_dir', help='The gcc dir') parser.add_argument('-t', '--target', required=True, dest='target', help=('The target to be build, the list is at' ' $(chromeos_root)/chromite/buildbot/cbuildbot' ' --list -all')) parser.add_argument('-l', '--local', action='store_true') parser.add_argument('-d', '--dest_dir', dest='dest_dir', help=('The dir to build the whole chromeos if' ' --local is set')) parser.add_argument('--chrome_version', dest='chrome_version', default='', help='The chrome version to use. ' 'Default it will use the latest one.') parser.add_argument('--chromeos_version', dest='chromeos_version', default='', help=('The chromeos version to use.' '(1) A release version in the format: ' "'\d+\.\d+\.\d+\.\d+.*'" "(2) 'latest_lkgm' for the latest lkgm version")) parser.add_argument('-r', '--replace_sysroot', action='store_true', help=('Whether or not to replace the build/$board dir' 'under the chroot of chromeos_root and copy ' 'the image to src/build/image/$board/latest.' ' Default is False')) parser.add_argument('-b', '--branch', dest='branch', default='', help=('The branch to run trybot, default is None')) parser.add_argument('-p', '--patch', dest='patch', default='', help=('The patches to be applied, the patches numbers ' "be seperated by ','")) script_dir = os.path.dirname(os.path.realpath(__file__)) args = parser.parse_args(argv[1:]) target = args.target if args.patch: patch = args.patch.split(',') else: patch = [] chromeos_root = misc.CanonicalizePath(args.chromeos_root) if args.chromeos_version and args.branch: raise RuntimeError( 'You can not set chromeos_version and branch at the ' 'same time.') manifests = None if args.branch: chromeos_version = '' branch = args.branch else: chromeos_version = args.chromeos_version manifests = manifest_versions.ManifestVersions() if chromeos_version == 'latest_lkgm': chromeos_version = manifests.TimeToVersion( time.mktime(time.gmtime())) logger.GetLogger().LogOutput('found version %s for latest LKGM' % (chromeos_version)) # TODO: this script currently does not handle the case where the version # is not in the "master" branch branch = 'master' if chromeos_version: manifest_patch = GetManifestPatch(manifests, chromeos_version, chromeos_root) patch.append(manifest_patch) if args.gcc_dir: # TODO: everytime we invoke this script we are getting a different # patch for GCC even if GCC has not changed. The description should # be based on the MD5 of the GCC patch contents. patch.append(UploadGccPatch(chromeos_root, args.gcc_dir, branch)) description = RunRemote(chromeos_root, branch, patch, args.local, target, args.chrome_version, args.dest_dir) if args.local or not args.dest_dir: # TODO: We are not checktng the result of cbuild_bot in here! return 0 # return value: # 0 => build bot was successful and image was put where requested # 1 => Build bot FAILED but image was put where requested # 2 => Build bot failed or BUild bot was successful but and image was # not generated or could not be put where expected os.chdir(script_dir) dest_dir = misc.CanonicalizePath(args.dest_dir) (bot_result, build_id) = FindBuildId(description) if bot_result > 0 and build_id > 0: logger.GetLogger().LogError( 'Remote trybot failed but image was generated') bot_result = 1 elif bot_result > 0: logger.GetLogger().LogError( 'Remote trybot failed. No image was generated') return 2 if 'toolchain' in branch: chromeos_version = FindVersionForToolchain(branch, chromeos_root) assert not manifest_versions.IsRFormatCrosVersion(chromeos_version) DownloadImage(target, build_id, dest_dir, chromeos_version) ret = UnpackImage(dest_dir) if ret != 0: return 2 # todo: return a more inteligent return value if not args.replace_sysroot: return bot_result ret = ReplaceSysroot(chromeos_root, args.dest_dir, target) if ret != 0: return 2 # got an image and we were successful in placing it where requested return bot_result
def __init__(self, name, remote, working_directory, chromeos_root, cache_conditions, labels, benchmarks, experiment_file, email_to, acquire_timeout, log_dir, log_level, share_cache, results_directory, locks_directory): self.name = name self.working_directory = working_directory self.remote = remote self.chromeos_root = chromeos_root self.cache_conditions = cache_conditions self.experiment_file = experiment_file self.email_to = email_to if not results_directory: self.results_directory = os.path.join(self.working_directory, self.name + '_results') else: self.results_directory = misc.CanonicalizePath(results_directory) self.log_dir = log_dir self.log_level = log_level self.labels = labels self.benchmarks = benchmarks self.num_complete = 0 self.num_run_complete = 0 self.share_cache = share_cache self.active_threads = [] # If locks_directory (self.lock_dir) not blank, we will use the file # locking mechanism; if it is blank then we will use the AFE server # locking mechanism. self.locks_dir = locks_directory self.locked_machines = [] if not remote: raise RuntimeError('No remote hosts specified') if not self.benchmarks: raise RuntimeError('No benchmarks specified') if not self.labels: raise RuntimeError('No labels specified') # We need one chromeos_root to run the benchmarks in, but it doesn't # matter where it is, unless the ABIs are different. if not chromeos_root: for label in self.labels: if label.chromeos_root: chromeos_root = label.chromeos_root break if not chromeos_root: raise RuntimeError('No chromeos_root given and could not determine ' 'one from the image path.') machine_manager_fn = MachineManager if test_flag.GetTestMode(): machine_manager_fn = MockMachineManager self.machine_manager = machine_manager_fn(chromeos_root, acquire_timeout, log_level, locks_directory) self.l = logger.GetLogger(log_dir) for machine in self.remote: # machine_manager.AddMachine only adds reachable machines. self.machine_manager.AddMachine(machine) # Now machine_manager._all_machines contains a list of reachable # machines. This is a subset of self.remote. We make both lists the same. self.remote = [m.name for m in self.machine_manager.GetAllMachines()] if not self.remote: raise RuntimeError('No machine available for running experiment.') for label in labels: # We filter out label remotes that are not reachable (not in # self.remote). So each label.remote is a sublist of experiment.remote. label.remote = [r for r in label.remote if r in self.remote] try: self.machine_manager.ComputeCommonCheckSum(label) except BadChecksum: # Force same image on all machines, then we do checksum again. No # bailout if checksums still do not match. self.machine_manager.ForceSameImageToAllMachines(label) self.machine_manager.ComputeCommonCheckSum(label) self.machine_manager.ComputeCommonCheckSumString(label) self.start_time = None self.benchmark_runs = self._GenerateBenchmarkRuns() self._schedv2 = None self._internal_counter_lock = Lock()
def Main(argv): """The main function.""" # Common initializations parser = argparse.ArgumentParser() parser.add_argument('-c', '--chromeos_root', dest='chromeos_root', default='../../', help=('ChromeOS root checkout directory' ' uses ../.. if none given.')) parser.add_argument('-g', '--gcc_dir', dest='gcc_dir', help='The directory where gcc resides.') parser.add_argument('--binutils_dir', dest='binutils_dir', help='The directory where binutils resides.') parser.add_argument('-x', '--gdb_dir', dest='gdb_dir', help='The directory where gdb resides.') parser.add_argument('-b', '--board', dest='board', default='x86-alex', help='The target board.') parser.add_argument('-n', '--noincremental', dest='noincremental', default=False, action='store_true', help='Use FEATURES=keepwork to do incremental builds.') parser.add_argument('--cflags', dest='cflags', default='', help='Build a compiler with specified CFLAGS') parser.add_argument('--cxxflags', dest='cxxflags', default='', help='Build a compiler with specified CXXFLAGS') parser.add_argument('--cflags_for_target', dest='cflags_for_target', default='', help='Build the target libraries with specified flags') parser.add_argument('--cxxflags_for_target', dest='cxxflags_for_target', default='', help='Build the target libraries with specified flags') parser.add_argument('--ldflags', dest='ldflags', default='', help='Build a compiler with specified LDFLAGS') parser.add_argument('-d', '--debug', dest='debug', default=False, action='store_true', help='Build a compiler with -g3 -O0 appended to both' ' CFLAGS and CXXFLAGS.') parser.add_argument('-m', '--mount_only', dest='mount_only', default=False, action='store_true', help='Just mount the tool directories.') parser.add_argument('-u', '--unmount_only', dest='unmount_only', default=False, action='store_true', help='Just unmount the tool directories.') parser.add_argument( '--extra_use_flags', dest='extra_use_flags', default='', help='Extra flag for USE, to be passed to the ebuild. ' "('multislot' and 'mounted_<tool>' are always passed.)") parser.add_argument('--gcc_enable_ccache', dest='gcc_enable_ccache', default=False, action='store_true', help='Enable ccache for the gcc invocations') options = parser.parse_args(argv) chromeos_root = misc.CanonicalizePath(options.chromeos_root) if options.gcc_dir: gcc_dir = misc.CanonicalizePath(options.gcc_dir) assert gcc_dir and os.path.isdir(gcc_dir), 'gcc_dir does not exist!' if options.binutils_dir: binutils_dir = misc.CanonicalizePath(options.binutils_dir) assert os.path.isdir(binutils_dir), 'binutils_dir does not exist!' if options.gdb_dir: gdb_dir = misc.CanonicalizePath(options.gdb_dir) assert os.path.isdir(gdb_dir), 'gdb_dir does not exist!' if options.unmount_only: options.mount_only = False elif options.mount_only: options.unmount_only = False build_env = {} if options.cflags: build_env['CFLAGS'] = '`portageq envvar CFLAGS` ' + options.cflags if options.cxxflags: build_env[ 'CXXFLAGS'] = '`portageq envvar CXXFLAGS` ' + options.cxxflags if options.cflags_for_target: build_env['CFLAGS_FOR_TARGET'] = options.cflags_for_target if options.cxxflags_for_target: build_env['CXXFLAGS_FOR_TARGET'] = options.cxxflags_for_target if options.ldflags: build_env['LDFLAGS'] = options.ldflags if options.debug: debug_flags = '-g3 -O0' if 'CFLAGS' in build_env: build_env['CFLAGS'] += ' %s' % (debug_flags) else: build_env['CFLAGS'] = debug_flags if 'CXXFLAGS' in build_env: build_env['CXXFLAGS'] += ' %s' % (debug_flags) else: build_env['CXXFLAGS'] = debug_flags if options.extra_use_flags: build_env['USE'] = options.extra_use_flags # Create toolchain parts toolchain_parts = {} for board in options.board.split(','): if options.gcc_dir: tp = ToolchainPart('gcc', gcc_dir, chromeos_root, board, not options.noincremental, build_env, options.gcc_enable_ccache) toolchain_parts[tp.tag] = tp tp.RunSetupBoardIfNecessary() if options.binutils_dir: tp = ToolchainPart('binutils', binutils_dir, chromeos_root, board, not options.noincremental, build_env) toolchain_parts[tp.tag] = tp tp.RunSetupBoardIfNecessary() if options.gdb_dir: tp = ToolchainPart('gdb', gdb_dir, chromeos_root, board, not options.noincremental, build_env) toolchain_parts[tp.tag] = tp tp.RunSetupBoardIfNecessary() rv = 0 try: for tag in toolchain_parts: tp = toolchain_parts[tag] if options.mount_only or options.unmount_only: tp.MountSources(options.unmount_only) else: rv = rv + tp.Build() finally: print('Exiting...') return rv