def BundleAFDOGenerationArtifacts(input_proto, output_proto, _config):
"""Generic function for creating tarballs of both AFDO and orderfile.
Args:
input_proto (BundleChromeAFDORequest): The input proto.
output_proto (BundleResponse): The output proto.
_config (api_config.ApiConfig): The API call config.
"""
chrome_root = input_proto.chroot.chrome_dir
output_dir = input_proto.output_dir
artifact_type = input_proto.artifact_type
build_target = controller_util.ParseBuildTarget(input_proto.build_target)
chroot = controller_util.ParseChroot(input_proto.chroot)
try:
is_orderfile = bool(artifact_type is toolchain_pb2.ORDERFILE)
results = artifacts.BundleAFDOGenerationArtifacts(
is_orderfile, chroot, chrome_root, build_target, output_dir)
except artifacts.Error as e:
cros_build_lib.Die(
'Error %s raised in BundleSimpleChromeArtifacts: %s', type(e), e)
for file_name in results:
output_proto.artifacts.add().path = file_name
def testListResponse(self):
"""Test calls helper method with correct args."""
mock_get_deps = self.PatchObject(
dependency_service, 'GetDependencies', return_value=['foo/bar-1.2.3'])
sysroot = sysroot_pb2.Sysroot(
path=self.sysroot, build_target=self.build_target)
path = '/path'
package = common_pb2.PackageInfo(category='foo', package_name='bar')
input_proto = depgraph_pb2.ListRequest(
sysroot=sysroot,
src_paths=[
depgraph_pb2.SourcePath(path='/path'),
],
packages=[package])
dependency.List(input_proto, self.response, self.api_config)
mock_get_deps.assert_called_once_with(
self.sysroot,
build_target=controller_util.ParseBuildTarget(self.build_target),
src_paths=[path],
packages=[controller_util.PackageInfoToCPV(package)])
expected_deps = [
common_pb2.PackageInfo(
category='foo', package_name='bar', version='1.2.3')
]
self.assertCountEqual(expected_deps, self.response.package_deps)
def Create(input_proto, output_proto, _config):
"""Create or replace a sysroot."""
update_chroot = not input_proto.flags.chroot_current
replace_sysroot = input_proto.flags.replace
build_target = controller_util.ParseBuildTarget(input_proto.build_target,
input_proto.profile)
package_indexes = [
binpkg.PackageIndexInfo.from_protobuf(x)
for x in input_proto.package_indexes
]
run_configs = sysroot.SetupBoardRunConfig(force=replace_sysroot,
upgrade_chroot=update_chroot,
package_indexes=package_indexes)
try:
created = sysroot.Create(build_target,
run_configs,
accept_licenses=_ACCEPTED_LICENSES)
except sysroot.Error as e:
cros_build_lib.Die(e)
output_proto.sysroot.path = created.path
output_proto.sysroot.build_target.name = build_target.name
return controller.RETURN_CODE_SUCCESS
def ExportCpeReport(input_proto, output_proto, config):
"""Export a CPE report.
Args:
input_proto (BundleRequest): The input proto.
output_proto (BundleResponse): The output proto.
config (api_config.ApiConfig): The API call config.
"""
chroot = controller_util.ParseChroot(input_proto.chroot)
output_dir = input_proto.output_dir
if input_proto.build_target.name:
# Legacy handling - use the default sysroot path for the build target.
build_target = controller_util.ParseBuildTarget(input_proto.build_target)
sysroot = sysroot_lib.Sysroot(build_target.root)
elif input_proto.sysroot.path:
sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
else:
# TODO(saklein): Switch to validate decorators once legacy handling can be
# cleaned up.
cros_build_lib.Die('sysroot.path is required.')
if config.validate_only:
return controller.RETURN_CODE_VALID_INPUT
cpe_result = artifacts.GenerateCpeReport(chroot, sysroot, output_dir)
output_proto.artifacts.add().path = cpe_result.report
output_proto.artifacts.add().path = cpe_result.warnings
def InstallToolchain(input_proto, output_proto, _config):
"""Install the toolchain into a sysroot."""
compile_source = (input_proto.flags.compile_source
or input_proto.flags.toolchain_changed)
sysroot_path = input_proto.sysroot.path
build_target = controller_util.ParseBuildTarget(
input_proto.sysroot.build_target)
target_sysroot = sysroot_lib.Sysroot(sysroot_path)
run_configs = sysroot.SetupBoardRunConfig(usepkg=not compile_source)
_LogBinhost(build_target.name)
try:
sysroot.InstallToolchain(build_target, target_sysroot, run_configs)
except sysroot_lib.ToolchainInstallError as e:
# Error installing - populate the failed package info.
for package in e.failed_toolchain_info:
package_info = output_proto.failed_packages.add()
controller_util.CPVToPackageInfo(package, package_info)
return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
return controller.RETURN_CODE_SUCCESS
def List(input_proto: depgraph_pb2.ListRequest,
output_proto: depgraph_pb2.ListResponse,
_config: api_config.ApiConfig):
"""Get a list of package dependencies.
Args:
input_proto: The input arguments message.
output_proto: The empty output message.
_config: The API call config.
"""
build_target = controller_util.ParseBuildTarget(
input_proto.sysroot.build_target)
sysroot_path = input_proto.sysroot.path
src_paths = [src_path.path for src_path in input_proto.src_paths]
packages = [
controller_util.PackageInfoToCPV(x) for x in input_proto.packages
]
package_deps = dependency.GetDependencies(sysroot_path,
build_target=build_target,
src_paths=src_paths,
packages=packages)
for package in package_deps:
pkg_info = output_proto.package_deps.add()
cpv = package_info.SplitCPV(package, strict=False)
controller_util.CPVToPackageInfo(cpv, pkg_info)
def BundleSimpleChromeArtifacts(input_proto, output_proto, _config):
"""Create the simple chrome artifacts."""
sysroot_path = input_proto.sysroot.path
output_dir = input_proto.output_dir
# Build out the argument instances.
build_target = controller_util.ParseBuildTarget(
input_proto.sysroot.build_target)
chroot = controller_util.ParseChroot(input_proto.chroot)
# Sysroot.path needs to be the fully qualified path, including the chroot.
full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep))
sysroot = sysroot_lib.Sysroot(full_sysroot_path)
# Quick sanity check that the sysroot exists before we go on.
if not sysroot.Exists():
cros_build_lib.Die('The sysroot does not exist.')
try:
results = artifacts.BundleSimpleChromeArtifacts(
chroot, sysroot, build_target, output_dir)
except artifacts.Error as e:
cros_build_lib.Die(
'Error %s raised in BundleSimpleChromeArtifacts: %s', type(e), e)
for file_name in results:
output_proto.artifacts.add().path = file_name
def testSuccess(self):
"""Test successful handling case."""
name = 'board'
build_target_message = common_pb2.BuildTarget(name=name)
expected = BuildTarget(name)
result = controller_util.ParseBuildTarget(build_target_message)
self.assertEqual(expected, result)
def BuildTargetUnitTest(input_proto, output_proto, _config):
"""Run a build target's ebuild unit tests."""
# Required args.
result_path = input_proto.result_path
# Method flags.
# An empty sysroot means build packages was not run. This is used for
# certain boards that need to use prebuilts (e.g. grunt's unittest-only).
was_built = not input_proto.flags.empty_sysroot
# Packages to be tested.
packages_package_info = input_proto.packages
packages = []
for package_info_msg in packages_package_info:
packages.append(controller_util.PackageInfoToString(package_info_msg))
# Skipped tests.
# TODO: Remove blacklist when we fully switch to blocklist.
blocklisted_package_info = (input_proto.package_blacklist
or input_proto.package_blocklist)
blocklist = []
for package_info_msg in blocklisted_package_info:
blocklist.append(controller_util.PackageInfoToString(package_info_msg))
# Allow call to succeed if no tests were found.
testable_packages_optional = input_proto.flags.testable_packages_optional
build_target = controller_util.ParseBuildTarget(input_proto.build_target)
chroot = controller_util.ParseChroot(input_proto.chroot)
code_coverage = input_proto.flags.code_coverage
result = test.BuildTargetUnitTest(
build_target,
chroot,
packages=packages,
blocklist=blocklist,
was_built=was_built,
code_coverage=code_coverage,
testable_packages_optional=testable_packages_optional)
if not result.success:
# Failed to run tests or some tests failed.
# Record all failed packages.
for cpv in result.failed_cpvs:
package_info_msg = output_proto.failed_packages.add()
controller_util.CPVToPackageInfo(cpv, package_info_msg)
if result.failed_cpvs:
return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
else:
return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
sysroot = sysroot_lib.Sysroot(build_target.root)
tarball = test.BuildTargetUnitTestTarball(chroot, sysroot, result_path)
if tarball:
output_proto.tarball_path = tarball
deserialize_metrics_log(output_proto.events, prefix=build_target.name)
def GetBestVisible(input_proto, output_proto, _config):
"""Returns the best visible PackageInfo for the indicated atom."""
build_target = None
if input_proto.build_target.name:
build_target = controller_util.ParseBuildTarget(input_proto.build_target)
cpv = packages.get_best_visible(input_proto.atom, build_target=build_target)
package_info = common_pb2.PackageInfo()
controller_util.CPVToPackageInfo(cpv, package_info)
output_proto.package_info.CopyFrom(package_info)
def GetBinhosts(input_proto, output_proto, _config):
"""Get a list of binhosts."""
build_target = controller_util.ParseBuildTarget(input_proto.build_target)
binhosts = binhost.GetBinhosts(build_target)
for current in binhosts:
new_binhost = output_proto.binhosts.add()
new_binhost.uri = current
new_binhost.package_index = 'Packages'
def PrepareBinhostUploads(input_proto, output_proto, config):
"""Return a list of files to upload to the binhost.
See BinhostService documentation in api/proto/binhost.proto.
Args:
input_proto (PrepareBinhostUploadsRequest): The input proto.
output_proto (PrepareBinhostUploadsResponse): The output proto.
config (api_config.ApiConfig): The API call config.
"""
if input_proto.sysroot.build_target.name:
build_target_msg = input_proto.sysroot.build_target
else:
build_target_msg = input_proto.build_target
sysroot_path = input_proto.sysroot.path
if not sysroot_path and not build_target_msg.name:
cros_build_lib.Die('Sysroot.path is required.')
build_target = controller_util.ParseBuildTarget(build_target_msg)
chroot = controller_util.ParseChroot(input_proto.chroot)
if not sysroot_path:
sysroot_path = build_target.root
sysroot = sysroot_lib.Sysroot(sysroot_path)
uri = input_proto.uri
# For now, we enforce that all input URIs are Google Storage buckets.
if not gs.PathIsGs(uri):
raise ValueError('Upload URI %s must be Google Storage.' % uri)
if config.validate_only:
return controller.RETURN_CODE_VALID_INPUT
parsed_uri = urllib.parse.urlparse(uri)
upload_uri = gs.GetGsURL(parsed_uri.netloc, for_gsutil=True).rstrip('/')
upload_path = parsed_uri.path.lstrip('/')
# Read all packages and update the index. The index must be uploaded to the
# binhost for Portage to use it, so include it in upload_targets.
uploads_dir = binhost.GetPrebuiltsRoot(chroot, sysroot, build_target)
index_path = binhost.UpdatePackageIndex(uploads_dir,
upload_uri,
upload_path,
sudo=True)
upload_targets = binhost.GetPrebuiltsFiles(uploads_dir)
assert index_path.startswith(uploads_dir), (
'expected index_path to start with uploads_dir')
upload_targets.append(index_path[len(uploads_dir):])
output_proto.uploads_dir = uploads_dir
for upload_target in upload_targets:
output_proto.upload_targets.add().path = upload_target.strip('/')
def BundleImageArchives(input_proto, output_proto, _config):
"""Create a .tar.xz archive for each image that has been created."""
build_target = controller_util.ParseBuildTarget(input_proto.build_target)
output_dir = input_proto.output_dir
image_dir = _GetImageDir(constants.SOURCE_ROOT, build_target.name)
if image_dir is None:
return
archives = artifacts.ArchiveImages(image_dir, output_dir)
for archive in archives:
output_proto.artifacts.add().path = os.path.join(output_dir, archive)
def testParseProfile(self):
"""Test the parsing of a profile."""
name = 'build-target-name'
profile = 'profile'
build_target_msg = common_pb2.BuildTarget(name=name)
profile_msg = sysroot_pb2.Profile(name=profile)
expected = build_target_lib.BuildTarget(name, profile=profile)
result = controller_util.ParseBuildTarget(build_target_msg,
profile_message=profile_msg)
self.assertEqual(expected, result)
def GetPrivatePrebuiltAclArgs(input_proto, output_proto, _config):
"""Get the ACL args from the files in the private overlays."""
build_target = controller_util.ParseBuildTarget(input_proto.build_target)
try:
args = binhost.GetPrebuiltAclArgs(build_target)
except binhost.Error as e:
cros_build_lib.Die(e)
for arg, value in args:
new_arg = output_proto.args.add()
new_arg.arg = arg
new_arg.value = value
def InstallPackages(input_proto, output_proto, _config):
"""Install packages into a sysroot, building as necessary and permitted."""
compile_source = input_proto.flags.compile_source
event_file = input_proto.flags.event_file
use_goma = input_proto.flags.use_goma
target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
build_target = controller_util.ParseBuildTarget(
input_proto.sysroot.build_target)
packages = [
controller_util.PackageInfoToString(x) for x in input_proto.packages
]
if not target_sysroot.IsToolchainInstalled():
cros_build_lib.Die('Toolchain must first be installed.')
_LogBinhost(build_target.name)
use_flags = [u.flag for u in input_proto.use_flags]
build_packages_config = sysroot.BuildPackagesRunConfig(
event_file=event_file,
usepkg=not compile_source,
install_debug_symbols=True,
packages=packages,
use_flags=use_flags,
use_goma=use_goma)
try:
sysroot.BuildPackages(build_target, target_sysroot,
build_packages_config)
except sysroot_lib.PackageInstallError as e:
if not e.failed_packages:
# No packages to report, so just exit with an error code.
return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
# We need to report the failed packages.
for package in e.failed_packages:
package_info = output_proto.failed_packages.add()
controller_util.CPVToPackageInfo(package, package_info)
return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
# Read metric events log and pipe them into output_proto.events.
deserialize_metrics_log(output_proto.events, prefix=build_target.name)
def GetTargetVersions(input_proto, output_proto, _config):
"""Returns the target versions."""
build_target = controller_util.ParseBuildTarget(input_proto.build_target)
android_version = packages.determine_android_version([build_target])
if android_version:
output_proto.android_version = android_version
android_branch_version = packages.determine_android_branch(build_target)
if android_branch_version:
output_proto.android_branch_version = android_branch_version
android_target_version = packages.determine_android_target(build_target)
if android_target_version:
output_proto.android_target_version = android_target_version
# TODO(crbug/1019770): Investigate cases where builds_chrome is true but
# chrome_version is None.
builds_chrome = packages.builds(constants.CHROME_CP, build_target)
if builds_chrome:
chrome_version = packages.determine_chrome_version(build_target)
if chrome_version:
output_proto.chrome_version = chrome_version
output_proto.platform_version = packages.determine_platform_version()
output_proto.milestone_version = packages.determine_milestone_version()
output_proto.full_version = packages.determine_full_version()
def GetChromeVersion(input_proto, output_proto, _config):
"""Returns the chrome version."""
build_target = controller_util.ParseBuildTarget(input_proto.build_target)
chrome_version = packages.determine_chrome_version(build_target)
if chrome_version:
output_proto.version = chrome_version
def InstallPackages(input_proto, output_proto, _config):
"""Install packages into a sysroot, building as necessary and permitted."""
compile_source = (input_proto.flags.compile_source
or input_proto.flags.toolchain_changed)
# Testing if Goma will support unknown compilers now.
use_goma = input_proto.flags.use_goma
target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
build_target = controller_util.ParseBuildTarget(
input_proto.sysroot.build_target)
# Get the package atom for each specified package. The field is optional, so
# error only when we cannot parse an atom for each of the given packages.
packages = [
controller_util.PackageInfoToCPV(x).cp for x in input_proto.packages
]
if input_proto.packages and not all(packages):
cros_build_lib.Die(
'Invalid package(s) specified. Unable to parse atom from all packages.'
)
package_indexes = [
binpkg.PackageIndexInfo.from_protobuf(x)
for x in input_proto.package_indexes
]
if not target_sysroot.IsToolchainInstalled():
cros_build_lib.Die('Toolchain must first be installed.')
_LogBinhost(build_target.name)
use_flags = [u.flag for u in input_proto.use_flags]
build_packages_config = sysroot.BuildPackagesRunConfig(
usepkg=not compile_source,
install_debug_symbols=True,
packages=packages,
package_indexes=package_indexes,
use_flags=use_flags,
use_goma=use_goma,
incremental_build=False)
try:
sysroot.BuildPackages(build_target, target_sysroot,
build_packages_config)
except sysroot_lib.PackageInstallError as e:
if not e.failed_packages:
# No packages to report, so just exit with an error code.
return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
# We need to report the failed packages.
for package in e.failed_packages:
package_info = output_proto.failed_packages.add()
controller_util.CPVToPackageInfo(package, package_info)
return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
# Copy goma logs to specified directory if there is a goma_config and
# it contains a log_dir to store artifacts.
if input_proto.goma_config.log_dir.dir:
# Get the goma log directory based on the GLOG_log_dir env variable.
# TODO(crbug.com/1045001): Replace environment variable with query to
# goma object after goma refactoring allows this.
log_source_dir = os.getenv('GLOG_log_dir')
if not log_source_dir:
cros_build_lib.Die('GLOG_log_dir must be defined.')
archiver = goma_lib.LogsArchiver(
log_source_dir,
dest_dir=input_proto.goma_config.log_dir.dir,
stats_file=input_proto.goma_config.stats_file,
counterz_file=input_proto.goma_config.counterz_file)
archiver_tuple = archiver.Archive()
if archiver_tuple.stats_file:
output_proto.goma_artifacts.stats_file = archiver_tuple.stats_file
if archiver_tuple.counterz_file:
output_proto.goma_artifacts.counterz_file = archiver_tuple.counterz_file
output_proto.goma_artifacts.log_files[:] = archiver_tuple.log_files
# Read metric events log and pipe them into output_proto.events.
deserialize_metrics_log(output_proto.events, prefix=build_target.name)
def HasChromePrebuilt(input_proto, output_proto, _config): """Checks if the most recent version of Chrome has a prebuilt.""" build_target = controller_util.ParseBuildTarget(input_proto.build_target) exists = packages.has_prebuilt(constants.CHROME_CP, build_target=build_target) output_proto.has_prebuilt = exists
def testWrongMessage(self):
"""Test invalid message type given."""
with self.assertRaises(AssertionError):
controller_util.ParseBuildTarget(
build_api_test_pb2.TestRequestMessage())
def BuildsChrome(input_proto, output_proto, _config): """Check if the board builds chrome.""" build_target = controller_util.ParseBuildTarget(input_proto.build_target) cpvs = [controller_util.PackageInfoToCPV(pi) for pi in input_proto.packages] builds_chrome = packages.builds(constants.CHROME_CP, build_target, cpvs) output_proto.builds_chrome = builds_chrome
def testWrongMessage(self):
"""Test invalid message type given."""
with self.assertRaises(AssertionError):
controller_util.ParseBuildTarget(common_pb2.Chroot())