def BundleAFDOGenerationArtifacts(input_proto, output_proto, _config): """Generic function for creating tarballs of both AFDO and orderfile. Args: input_proto (BundleChromeAFDORequest): The input proto. output_proto (BundleResponse): The output proto. _config (api_config.ApiConfig): The API call config. """ chrome_root = input_proto.chroot.chrome_dir output_dir = input_proto.output_dir artifact_type = input_proto.artifact_type build_target = controller_util.ParseBuildTarget(input_proto.build_target) chroot = controller_util.ParseChroot(input_proto.chroot) try: is_orderfile = bool(artifact_type is toolchain_pb2.ORDERFILE) results = artifacts.BundleAFDOGenerationArtifacts( is_orderfile, chroot, chrome_root, build_target, output_dir) except artifacts.Error as e: cros_build_lib.Die( 'Error %s raised in BundleSimpleChromeArtifacts: %s', type(e), e) for file_name in results: output_proto.artifacts.add().path = file_name
def testListResponse(self): """Test calls helper method with correct args.""" mock_get_deps = self.PatchObject( dependency_service, 'GetDependencies', return_value=['foo/bar-1.2.3']) sysroot = sysroot_pb2.Sysroot( path=self.sysroot, build_target=self.build_target) path = '/path' package = common_pb2.PackageInfo(category='foo', package_name='bar') input_proto = depgraph_pb2.ListRequest( sysroot=sysroot, src_paths=[ depgraph_pb2.SourcePath(path='/path'), ], packages=[package]) dependency.List(input_proto, self.response, self.api_config) mock_get_deps.assert_called_once_with( self.sysroot, build_target=controller_util.ParseBuildTarget(self.build_target), src_paths=[path], packages=[controller_util.PackageInfoToCPV(package)]) expected_deps = [ common_pb2.PackageInfo( category='foo', package_name='bar', version='1.2.3') ] self.assertCountEqual(expected_deps, self.response.package_deps)
def Create(input_proto, output_proto, _config): """Create or replace a sysroot.""" update_chroot = not input_proto.flags.chroot_current replace_sysroot = input_proto.flags.replace build_target = controller_util.ParseBuildTarget(input_proto.build_target, input_proto.profile) package_indexes = [ binpkg.PackageIndexInfo.from_protobuf(x) for x in input_proto.package_indexes ] run_configs = sysroot.SetupBoardRunConfig(force=replace_sysroot, upgrade_chroot=update_chroot, package_indexes=package_indexes) try: created = sysroot.Create(build_target, run_configs, accept_licenses=_ACCEPTED_LICENSES) except sysroot.Error as e: cros_build_lib.Die(e) output_proto.sysroot.path = created.path output_proto.sysroot.build_target.name = build_target.name return controller.RETURN_CODE_SUCCESS
def ExportCpeReport(input_proto, output_proto, config): """Export a CPE report. Args: input_proto (BundleRequest): The input proto. output_proto (BundleResponse): The output proto. config (api_config.ApiConfig): The API call config. """ chroot = controller_util.ParseChroot(input_proto.chroot) output_dir = input_proto.output_dir if input_proto.build_target.name: # Legacy handling - use the default sysroot path for the build target. build_target = controller_util.ParseBuildTarget(input_proto.build_target) sysroot = sysroot_lib.Sysroot(build_target.root) elif input_proto.sysroot.path: sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path) else: # TODO(saklein): Switch to validate decorators once legacy handling can be # cleaned up. cros_build_lib.Die('sysroot.path is required.') if config.validate_only: return controller.RETURN_CODE_VALID_INPUT cpe_result = artifacts.GenerateCpeReport(chroot, sysroot, output_dir) output_proto.artifacts.add().path = cpe_result.report output_proto.artifacts.add().path = cpe_result.warnings
def InstallToolchain(input_proto, output_proto, _config): """Install the toolchain into a sysroot.""" compile_source = (input_proto.flags.compile_source or input_proto.flags.toolchain_changed) sysroot_path = input_proto.sysroot.path build_target = controller_util.ParseBuildTarget( input_proto.sysroot.build_target) target_sysroot = sysroot_lib.Sysroot(sysroot_path) run_configs = sysroot.SetupBoardRunConfig(usepkg=not compile_source) _LogBinhost(build_target.name) try: sysroot.InstallToolchain(build_target, target_sysroot, run_configs) except sysroot_lib.ToolchainInstallError as e: # Error installing - populate the failed package info. for package in e.failed_toolchain_info: package_info = output_proto.failed_packages.add() controller_util.CPVToPackageInfo(package, package_info) return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE return controller.RETURN_CODE_SUCCESS
def List(input_proto: depgraph_pb2.ListRequest, output_proto: depgraph_pb2.ListResponse, _config: api_config.ApiConfig): """Get a list of package dependencies. Args: input_proto: The input arguments message. output_proto: The empty output message. _config: The API call config. """ build_target = controller_util.ParseBuildTarget( input_proto.sysroot.build_target) sysroot_path = input_proto.sysroot.path src_paths = [src_path.path for src_path in input_proto.src_paths] packages = [ controller_util.PackageInfoToCPV(x) for x in input_proto.packages ] package_deps = dependency.GetDependencies(sysroot_path, build_target=build_target, src_paths=src_paths, packages=packages) for package in package_deps: pkg_info = output_proto.package_deps.add() cpv = package_info.SplitCPV(package, strict=False) controller_util.CPVToPackageInfo(cpv, pkg_info)
def BundleSimpleChromeArtifacts(input_proto, output_proto, _config): """Create the simple chrome artifacts.""" sysroot_path = input_proto.sysroot.path output_dir = input_proto.output_dir # Build out the argument instances. build_target = controller_util.ParseBuildTarget( input_proto.sysroot.build_target) chroot = controller_util.ParseChroot(input_proto.chroot) # Sysroot.path needs to be the fully qualified path, including the chroot. full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep)) sysroot = sysroot_lib.Sysroot(full_sysroot_path) # Quick sanity check that the sysroot exists before we go on. if not sysroot.Exists(): cros_build_lib.Die('The sysroot does not exist.') try: results = artifacts.BundleSimpleChromeArtifacts( chroot, sysroot, build_target, output_dir) except artifacts.Error as e: cros_build_lib.Die( 'Error %s raised in BundleSimpleChromeArtifacts: %s', type(e), e) for file_name in results: output_proto.artifacts.add().path = file_name
def testSuccess(self): """Test successful handling case.""" name = 'board' build_target_message = common_pb2.BuildTarget(name=name) expected = BuildTarget(name) result = controller_util.ParseBuildTarget(build_target_message) self.assertEqual(expected, result)
def BuildTargetUnitTest(input_proto, output_proto, _config): """Run a build target's ebuild unit tests.""" # Required args. result_path = input_proto.result_path # Method flags. # An empty sysroot means build packages was not run. This is used for # certain boards that need to use prebuilts (e.g. grunt's unittest-only). was_built = not input_proto.flags.empty_sysroot # Packages to be tested. packages_package_info = input_proto.packages packages = [] for package_info_msg in packages_package_info: packages.append(controller_util.PackageInfoToString(package_info_msg)) # Skipped tests. # TODO: Remove blacklist when we fully switch to blocklist. blocklisted_package_info = (input_proto.package_blacklist or input_proto.package_blocklist) blocklist = [] for package_info_msg in blocklisted_package_info: blocklist.append(controller_util.PackageInfoToString(package_info_msg)) # Allow call to succeed if no tests were found. testable_packages_optional = input_proto.flags.testable_packages_optional build_target = controller_util.ParseBuildTarget(input_proto.build_target) chroot = controller_util.ParseChroot(input_proto.chroot) code_coverage = input_proto.flags.code_coverage result = test.BuildTargetUnitTest( build_target, chroot, packages=packages, blocklist=blocklist, was_built=was_built, code_coverage=code_coverage, testable_packages_optional=testable_packages_optional) if not result.success: # Failed to run tests or some tests failed. # Record all failed packages. for cpv in result.failed_cpvs: package_info_msg = output_proto.failed_packages.add() controller_util.CPVToPackageInfo(cpv, package_info_msg) if result.failed_cpvs: return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE else: return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY sysroot = sysroot_lib.Sysroot(build_target.root) tarball = test.BuildTargetUnitTestTarball(chroot, sysroot, result_path) if tarball: output_proto.tarball_path = tarball deserialize_metrics_log(output_proto.events, prefix=build_target.name)
def GetBestVisible(input_proto, output_proto, _config): """Returns the best visible PackageInfo for the indicated atom.""" build_target = None if input_proto.build_target.name: build_target = controller_util.ParseBuildTarget(input_proto.build_target) cpv = packages.get_best_visible(input_proto.atom, build_target=build_target) package_info = common_pb2.PackageInfo() controller_util.CPVToPackageInfo(cpv, package_info) output_proto.package_info.CopyFrom(package_info)
def GetBinhosts(input_proto, output_proto, _config): """Get a list of binhosts.""" build_target = controller_util.ParseBuildTarget(input_proto.build_target) binhosts = binhost.GetBinhosts(build_target) for current in binhosts: new_binhost = output_proto.binhosts.add() new_binhost.uri = current new_binhost.package_index = 'Packages'
def PrepareBinhostUploads(input_proto, output_proto, config): """Return a list of files to upload to the binhost. See BinhostService documentation in api/proto/binhost.proto. Args: input_proto (PrepareBinhostUploadsRequest): The input proto. output_proto (PrepareBinhostUploadsResponse): The output proto. config (api_config.ApiConfig): The API call config. """ if input_proto.sysroot.build_target.name: build_target_msg = input_proto.sysroot.build_target else: build_target_msg = input_proto.build_target sysroot_path = input_proto.sysroot.path if not sysroot_path and not build_target_msg.name: cros_build_lib.Die('Sysroot.path is required.') build_target = controller_util.ParseBuildTarget(build_target_msg) chroot = controller_util.ParseChroot(input_proto.chroot) if not sysroot_path: sysroot_path = build_target.root sysroot = sysroot_lib.Sysroot(sysroot_path) uri = input_proto.uri # For now, we enforce that all input URIs are Google Storage buckets. if not gs.PathIsGs(uri): raise ValueError('Upload URI %s must be Google Storage.' % uri) if config.validate_only: return controller.RETURN_CODE_VALID_INPUT parsed_uri = urllib.parse.urlparse(uri) upload_uri = gs.GetGsURL(parsed_uri.netloc, for_gsutil=True).rstrip('/') upload_path = parsed_uri.path.lstrip('/') # Read all packages and update the index. The index must be uploaded to the # binhost for Portage to use it, so include it in upload_targets. uploads_dir = binhost.GetPrebuiltsRoot(chroot, sysroot, build_target) index_path = binhost.UpdatePackageIndex(uploads_dir, upload_uri, upload_path, sudo=True) upload_targets = binhost.GetPrebuiltsFiles(uploads_dir) assert index_path.startswith(uploads_dir), ( 'expected index_path to start with uploads_dir') upload_targets.append(index_path[len(uploads_dir):]) output_proto.uploads_dir = uploads_dir for upload_target in upload_targets: output_proto.upload_targets.add().path = upload_target.strip('/')
def BundleImageArchives(input_proto, output_proto, _config): """Create a .tar.xz archive for each image that has been created.""" build_target = controller_util.ParseBuildTarget(input_proto.build_target) output_dir = input_proto.output_dir image_dir = _GetImageDir(constants.SOURCE_ROOT, build_target.name) if image_dir is None: return archives = artifacts.ArchiveImages(image_dir, output_dir) for archive in archives: output_proto.artifacts.add().path = os.path.join(output_dir, archive)
def testParseProfile(self): """Test the parsing of a profile.""" name = 'build-target-name' profile = 'profile' build_target_msg = common_pb2.BuildTarget(name=name) profile_msg = sysroot_pb2.Profile(name=profile) expected = build_target_lib.BuildTarget(name, profile=profile) result = controller_util.ParseBuildTarget(build_target_msg, profile_message=profile_msg) self.assertEqual(expected, result)
def GetPrivatePrebuiltAclArgs(input_proto, output_proto, _config): """Get the ACL args from the files in the private overlays.""" build_target = controller_util.ParseBuildTarget(input_proto.build_target) try: args = binhost.GetPrebuiltAclArgs(build_target) except binhost.Error as e: cros_build_lib.Die(e) for arg, value in args: new_arg = output_proto.args.add() new_arg.arg = arg new_arg.value = value
def InstallPackages(input_proto, output_proto, _config): """Install packages into a sysroot, building as necessary and permitted.""" compile_source = input_proto.flags.compile_source event_file = input_proto.flags.event_file use_goma = input_proto.flags.use_goma target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path) build_target = controller_util.ParseBuildTarget( input_proto.sysroot.build_target) packages = [ controller_util.PackageInfoToString(x) for x in input_proto.packages ] if not target_sysroot.IsToolchainInstalled(): cros_build_lib.Die('Toolchain must first be installed.') _LogBinhost(build_target.name) use_flags = [u.flag for u in input_proto.use_flags] build_packages_config = sysroot.BuildPackagesRunConfig( event_file=event_file, usepkg=not compile_source, install_debug_symbols=True, packages=packages, use_flags=use_flags, use_goma=use_goma) try: sysroot.BuildPackages(build_target, target_sysroot, build_packages_config) except sysroot_lib.PackageInstallError as e: if not e.failed_packages: # No packages to report, so just exit with an error code. return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY # We need to report the failed packages. for package in e.failed_packages: package_info = output_proto.failed_packages.add() controller_util.CPVToPackageInfo(package, package_info) return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE # Read metric events log and pipe them into output_proto.events. deserialize_metrics_log(output_proto.events, prefix=build_target.name)
def GetTargetVersions(input_proto, output_proto, _config): """Returns the target versions.""" build_target = controller_util.ParseBuildTarget(input_proto.build_target) android_version = packages.determine_android_version([build_target]) if android_version: output_proto.android_version = android_version android_branch_version = packages.determine_android_branch(build_target) if android_branch_version: output_proto.android_branch_version = android_branch_version android_target_version = packages.determine_android_target(build_target) if android_target_version: output_proto.android_target_version = android_target_version # TODO(crbug/1019770): Investigate cases where builds_chrome is true but # chrome_version is None. builds_chrome = packages.builds(constants.CHROME_CP, build_target) if builds_chrome: chrome_version = packages.determine_chrome_version(build_target) if chrome_version: output_proto.chrome_version = chrome_version output_proto.platform_version = packages.determine_platform_version() output_proto.milestone_version = packages.determine_milestone_version() output_proto.full_version = packages.determine_full_version()
def GetChromeVersion(input_proto, output_proto, _config): """Returns the chrome version.""" build_target = controller_util.ParseBuildTarget(input_proto.build_target) chrome_version = packages.determine_chrome_version(build_target) if chrome_version: output_proto.version = chrome_version
def InstallPackages(input_proto, output_proto, _config): """Install packages into a sysroot, building as necessary and permitted.""" compile_source = (input_proto.flags.compile_source or input_proto.flags.toolchain_changed) # Testing if Goma will support unknown compilers now. use_goma = input_proto.flags.use_goma target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path) build_target = controller_util.ParseBuildTarget( input_proto.sysroot.build_target) # Get the package atom for each specified package. The field is optional, so # error only when we cannot parse an atom for each of the given packages. packages = [ controller_util.PackageInfoToCPV(x).cp for x in input_proto.packages ] if input_proto.packages and not all(packages): cros_build_lib.Die( 'Invalid package(s) specified. Unable to parse atom from all packages.' ) package_indexes = [ binpkg.PackageIndexInfo.from_protobuf(x) for x in input_proto.package_indexes ] if not target_sysroot.IsToolchainInstalled(): cros_build_lib.Die('Toolchain must first be installed.') _LogBinhost(build_target.name) use_flags = [u.flag for u in input_proto.use_flags] build_packages_config = sysroot.BuildPackagesRunConfig( usepkg=not compile_source, install_debug_symbols=True, packages=packages, package_indexes=package_indexes, use_flags=use_flags, use_goma=use_goma, incremental_build=False) try: sysroot.BuildPackages(build_target, target_sysroot, build_packages_config) except sysroot_lib.PackageInstallError as e: if not e.failed_packages: # No packages to report, so just exit with an error code. return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY # We need to report the failed packages. for package in e.failed_packages: package_info = output_proto.failed_packages.add() controller_util.CPVToPackageInfo(package, package_info) return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE # Copy goma logs to specified directory if there is a goma_config and # it contains a log_dir to store artifacts. if input_proto.goma_config.log_dir.dir: # Get the goma log directory based on the GLOG_log_dir env variable. # TODO(crbug.com/1045001): Replace environment variable with query to # goma object after goma refactoring allows this. log_source_dir = os.getenv('GLOG_log_dir') if not log_source_dir: cros_build_lib.Die('GLOG_log_dir must be defined.') archiver = goma_lib.LogsArchiver( log_source_dir, dest_dir=input_proto.goma_config.log_dir.dir, stats_file=input_proto.goma_config.stats_file, counterz_file=input_proto.goma_config.counterz_file) archiver_tuple = archiver.Archive() if archiver_tuple.stats_file: output_proto.goma_artifacts.stats_file = archiver_tuple.stats_file if archiver_tuple.counterz_file: output_proto.goma_artifacts.counterz_file = archiver_tuple.counterz_file output_proto.goma_artifacts.log_files[:] = archiver_tuple.log_files # Read metric events log and pipe them into output_proto.events. deserialize_metrics_log(output_proto.events, prefix=build_target.name)
def HasChromePrebuilt(input_proto, output_proto, _config): """Checks if the most recent version of Chrome has a prebuilt.""" build_target = controller_util.ParseBuildTarget(input_proto.build_target) exists = packages.has_prebuilt(constants.CHROME_CP, build_target=build_target) output_proto.has_prebuilt = exists
def testWrongMessage(self): """Test invalid message type given.""" with self.assertRaises(AssertionError): controller_util.ParseBuildTarget( build_api_test_pb2.TestRequestMessage())
def BuildsChrome(input_proto, output_proto, _config): """Check if the board builds chrome.""" build_target = controller_util.ParseBuildTarget(input_proto.build_target) cpvs = [controller_util.PackageInfoToCPV(pi) for pi in input_proto.packages] builds_chrome = packages.builds(constants.CHROME_CP, build_target, cpvs) output_proto.builds_chrome = builds_chrome
def testWrongMessage(self): """Test invalid message type given.""" with self.assertRaises(AssertionError): controller_util.ParseBuildTarget(common_pb2.Chroot())