def Create(input_proto, output_proto, _config):
"""Build an image.
Args:
input_proto (image_pb2.CreateImageRequest): The input message.
output_proto (image_pb2.CreateImageResult): The output message.
_config (api_config.ApiConfig): The API call config.
"""
board = input_proto.build_target.name
# Build the base image if no images provided.
to_build = input_proto.image_types or [_BASE_ID]
image_types, vm_types = _ParseImagesToCreate(to_build)
build_config = _ParseCreateBuildConfig(input_proto)
# Sorted isn't really necessary here, but it's much easier to test.
result = image.Build(board=board, images=sorted(list(image_types)),
config=build_config)
output_proto.success = result.success
if result.success:
# Success -- we need to list out the images we built in the output.
_PopulateBuiltImages(board, image_types, output_proto)
if vm_types:
for vm_type in vm_types:
is_test = vm_type in [_TEST_VM_ID, _TEST_GUEST_VM_ID]
try:
if vm_type in [_BASE_GUEST_VM_ID, _TEST_GUEST_VM_ID]:
vm_path = image.CreateGuestVm(board, is_test=is_test)
else:
vm_path = image.CreateVm(
board, disk_layout=build_config.disk_layout, is_test=is_test)
except image.ImageToVmError as e:
cros_build_lib.Die(e)
new_image = output_proto.images.add()
new_image.path = vm_path
new_image.type = vm_type
new_image.build_target.name = board
# Read metric events log and pipe them into output_proto.events.
deserialize_metrics_log(output_proto.events, prefix=board)
return controller.RETURN_CODE_SUCCESS
else:
# Failure, include all of the failed packages in the output when available.
if not result.failed_packages:
return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
for package in result.failed_packages:
current = output_proto.failed_packages.add()
current.category = package.category
current.package_name = package.package
if package.version:
current.version = package.version
return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
def BuildTargetUnitTest(input_proto, output_proto, _config):
"""Run a build target's ebuild unit tests."""
# Required args.
result_path = input_proto.result_path
# Method flags.
# An empty sysroot means build packages was not run. This is used for
# certain boards that need to use prebuilts (e.g. grunt's unittest-only).
was_built = not input_proto.flags.empty_sysroot
# Packages to be tested.
packages_package_info = input_proto.packages
packages = []
for package_info_msg in packages_package_info:
packages.append(controller_util.PackageInfoToString(package_info_msg))
# Skipped tests.
# TODO: Remove blacklist when we fully switch to blocklist.
blocklisted_package_info = (input_proto.package_blacklist
or input_proto.package_blocklist)
blocklist = []
for package_info_msg in blocklisted_package_info:
blocklist.append(controller_util.PackageInfoToString(package_info_msg))
# Allow call to succeed if no tests were found.
testable_packages_optional = input_proto.flags.testable_packages_optional
build_target = controller_util.ParseBuildTarget(input_proto.build_target)
chroot = controller_util.ParseChroot(input_proto.chroot)
code_coverage = input_proto.flags.code_coverage
result = test.BuildTargetUnitTest(
build_target,
chroot,
packages=packages,
blocklist=blocklist,
was_built=was_built,
code_coverage=code_coverage,
testable_packages_optional=testable_packages_optional)
if not result.success:
# Failed to run tests or some tests failed.
# Record all failed packages.
for cpv in result.failed_cpvs:
package_info_msg = output_proto.failed_packages.add()
controller_util.CPVToPackageInfo(cpv, package_info_msg)
if result.failed_cpvs:
return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
else:
return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
sysroot = sysroot_lib.Sysroot(build_target.root)
tarball = test.BuildTargetUnitTestTarball(chroot, sysroot, result_path)
if tarball:
output_proto.tarball_path = tarball
deserialize_metrics_log(output_proto.events, prefix=build_target.name)
def testDeserializeGauge(self):
"""Test deserialization of a gauge."""
response = build_api_test_pb2.TestResultMessage()
mock_events = [
MetricEvent(1000, 'a.gauge', OP_GAUGE, arg=17),
]
with mock.patch('chromite.api.metrics.metrics.read_metrics_events',
return_value=mock_events):
metrics.deserialize_metrics_log(response.events)
self.assertEqual(len(response.events), 1)
self.assertEqual(response.events[0].name, 'a.gauge')
self.assertEqual(response.events[0].timestamp_milliseconds, 1000)
self.assertEqual(response.events[0].gauge, 17)
def testDeserializeTimer(self):
"""Test timer math and deserialization into proto objects."""
response = build_api_test_pb2.TestResultMessage()
mock_events = [
MetricEvent(600, 'a.b', OP_START_TIMER, arg='100'),
MetricEvent(1000, 'a.b', OP_STOP_TIMER, arg='100'),
]
with mock.patch('chromite.api.metrics.metrics.read_metrics_events',
return_value=mock_events):
metrics.deserialize_metrics_log(response.events)
self.assertEqual(len(response.events), 1)
self.assertEqual(response.events[0].name, 'a.b')
self.assertEqual(response.events[0].timestamp_milliseconds, 1000)
self.assertEqual(response.events[0].duration_milliseconds, 1000-600)
def testDeserializeNamedEvent(self):
"""Test deserialization of a named event.
This test also includes a prefix to test for proper prepending.
"""
response = build_api_test_pb2.TestResultMessage()
mock_events = [
MetricEvent(1000, 'a.named_event', OP_NAMED_EVENT, arg=None),
]
with mock.patch('chromite.api.metrics.metrics.read_metrics_events',
return_value=mock_events):
metrics.deserialize_metrics_log(response.events, prefix='prefix')
self.assertEqual(len(response.events), 1)
self.assertEqual(response.events[0].name, 'prefix.a.named_event')
self.assertEqual(response.events[0].timestamp_milliseconds, 1000)
self.assertFalse(response.events[0].duration_milliseconds)
def InstallPackages(input_proto, output_proto, _config):
"""Install packages into a sysroot, building as necessary and permitted."""
compile_source = input_proto.flags.compile_source
event_file = input_proto.flags.event_file
use_goma = input_proto.flags.use_goma
target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
build_target = controller_util.ParseBuildTarget(
input_proto.sysroot.build_target)
packages = [
controller_util.PackageInfoToString(x) for x in input_proto.packages
]
if not target_sysroot.IsToolchainInstalled():
cros_build_lib.Die('Toolchain must first be installed.')
_LogBinhost(build_target.name)
use_flags = [u.flag for u in input_proto.use_flags]
build_packages_config = sysroot.BuildPackagesRunConfig(
event_file=event_file,
usepkg=not compile_source,
install_debug_symbols=True,
packages=packages,
use_flags=use_flags,
use_goma=use_goma)
try:
sysroot.BuildPackages(build_target, target_sysroot,
build_packages_config)
except sysroot_lib.PackageInstallError as e:
if not e.failed_packages:
# No packages to report, so just exit with an error code.
return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
# We need to report the failed packages.
for package in e.failed_packages:
package_info = output_proto.failed_packages.add()
controller_util.CPVToPackageInfo(package, package_info)
return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
# Read metric events log and pipe them into output_proto.events.
deserialize_metrics_log(output_proto.events, prefix=build_target.name)
def BuildTargetUnitTest(input_proto, output_proto, _config):
"""Run a build target's ebuild unit tests."""
# Required args.
board = input_proto.build_target.name
result_path = input_proto.result_path
# Method flags.
# An empty sysroot means build packages was not run. This is used for
# certain boards that need to use prebuilts (e.g. grunt's unittest-only).
was_built = not input_proto.flags.empty_sysroot
# Skipped tests.
blacklisted_package_info = input_proto.package_blacklist
blacklist = []
for package_info in blacklisted_package_info:
blacklist.append(controller_util.PackageInfoToString(package_info))
build_target = build_target_util.BuildTarget(board)
chroot = controller_util.ParseChroot(input_proto.chroot)
result = test.BuildTargetUnitTest(build_target,
chroot,
blacklist=blacklist,
was_built=was_built)
if not result.success:
# Failed to run tests or some tests failed.
# Record all failed packages.
for cpv in result.failed_cpvs:
package_info = output_proto.failed_packages.add()
controller_util.CPVToPackageInfo(cpv, package_info)
if result.failed_cpvs:
return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
else:
return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
sysroot = sysroot_lib.Sysroot(build_target.root)
tarball = test.BuildTargetUnitTestTarball(chroot, sysroot, result_path)
if tarball:
output_proto.tarball_path = tarball
deserialize_metrics_log(output_proto.events, prefix=build_target.name)
def InstallPackages(input_proto, output_proto, _config):
"""Install packages into a sysroot, building as necessary and permitted."""
compile_source = (input_proto.flags.compile_source
or input_proto.flags.toolchain_changed)
# Testing if Goma will support unknown compilers now.
use_goma = input_proto.flags.use_goma
target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
build_target = controller_util.ParseBuildTarget(
input_proto.sysroot.build_target)
# Get the package atom for each specified package. The field is optional, so
# error only when we cannot parse an atom for each of the given packages.
packages = [
controller_util.PackageInfoToCPV(x).cp for x in input_proto.packages
]
if input_proto.packages and not all(packages):
cros_build_lib.Die(
'Invalid package(s) specified. Unable to parse atom from all packages.'
)
package_indexes = [
binpkg.PackageIndexInfo.from_protobuf(x)
for x in input_proto.package_indexes
]
if not target_sysroot.IsToolchainInstalled():
cros_build_lib.Die('Toolchain must first be installed.')
_LogBinhost(build_target.name)
use_flags = [u.flag for u in input_proto.use_flags]
build_packages_config = sysroot.BuildPackagesRunConfig(
usepkg=not compile_source,
install_debug_symbols=True,
packages=packages,
package_indexes=package_indexes,
use_flags=use_flags,
use_goma=use_goma,
incremental_build=False)
try:
sysroot.BuildPackages(build_target, target_sysroot,
build_packages_config)
except sysroot_lib.PackageInstallError as e:
if not e.failed_packages:
# No packages to report, so just exit with an error code.
return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
# We need to report the failed packages.
for package in e.failed_packages:
package_info = output_proto.failed_packages.add()
controller_util.CPVToPackageInfo(package, package_info)
return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
# Copy goma logs to specified directory if there is a goma_config and
# it contains a log_dir to store artifacts.
if input_proto.goma_config.log_dir.dir:
# Get the goma log directory based on the GLOG_log_dir env variable.
# TODO(crbug.com/1045001): Replace environment variable with query to
# goma object after goma refactoring allows this.
log_source_dir = os.getenv('GLOG_log_dir')
if not log_source_dir:
cros_build_lib.Die('GLOG_log_dir must be defined.')
archiver = goma_lib.LogsArchiver(
log_source_dir,
dest_dir=input_proto.goma_config.log_dir.dir,
stats_file=input_proto.goma_config.stats_file,
counterz_file=input_proto.goma_config.counterz_file)
archiver_tuple = archiver.Archive()
if archiver_tuple.stats_file:
output_proto.goma_artifacts.stats_file = archiver_tuple.stats_file
if archiver_tuple.counterz_file:
output_proto.goma_artifacts.counterz_file = archiver_tuple.counterz_file
output_proto.goma_artifacts.log_files[:] = archiver_tuple.log_files
# Read metric events log and pipe them into output_proto.events.
deserialize_metrics_log(output_proto.events, prefix=build_target.name)
