def Create(input_proto, output_proto, _config): """Build an image. Args: input_proto (image_pb2.CreateImageRequest): The input message. output_proto (image_pb2.CreateImageResult): The output message. _config (api_config.ApiConfig): The API call config. """ board = input_proto.build_target.name # Build the base image if no images provided. to_build = input_proto.image_types or [_BASE_ID] image_types, vm_types = _ParseImagesToCreate(to_build) build_config = _ParseCreateBuildConfig(input_proto) # Sorted isn't really necessary here, but it's much easier to test. result = image.Build(board=board, images=sorted(list(image_types)), config=build_config) output_proto.success = result.success if result.success: # Success -- we need to list out the images we built in the output. _PopulateBuiltImages(board, image_types, output_proto) if vm_types: for vm_type in vm_types: is_test = vm_type in [_TEST_VM_ID, _TEST_GUEST_VM_ID] try: if vm_type in [_BASE_GUEST_VM_ID, _TEST_GUEST_VM_ID]: vm_path = image.CreateGuestVm(board, is_test=is_test) else: vm_path = image.CreateVm( board, disk_layout=build_config.disk_layout, is_test=is_test) except image.ImageToVmError as e: cros_build_lib.Die(e) new_image = output_proto.images.add() new_image.path = vm_path new_image.type = vm_type new_image.build_target.name = board # Read metric events log and pipe them into output_proto.events. deserialize_metrics_log(output_proto.events, prefix=board) return controller.RETURN_CODE_SUCCESS else: # Failure, include all of the failed packages in the output when available. if not result.failed_packages: return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY for package in result.failed_packages: current = output_proto.failed_packages.add() current.category = package.category current.package_name = package.package if package.version: current.version = package.version return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
def BuildTargetUnitTest(input_proto, output_proto, _config): """Run a build target's ebuild unit tests.""" # Required args. result_path = input_proto.result_path # Method flags. # An empty sysroot means build packages was not run. This is used for # certain boards that need to use prebuilts (e.g. grunt's unittest-only). was_built = not input_proto.flags.empty_sysroot # Packages to be tested. packages_package_info = input_proto.packages packages = [] for package_info_msg in packages_package_info: packages.append(controller_util.PackageInfoToString(package_info_msg)) # Skipped tests. # TODO: Remove blacklist when we fully switch to blocklist. blocklisted_package_info = (input_proto.package_blacklist or input_proto.package_blocklist) blocklist = [] for package_info_msg in blocklisted_package_info: blocklist.append(controller_util.PackageInfoToString(package_info_msg)) # Allow call to succeed if no tests were found. testable_packages_optional = input_proto.flags.testable_packages_optional build_target = controller_util.ParseBuildTarget(input_proto.build_target) chroot = controller_util.ParseChroot(input_proto.chroot) code_coverage = input_proto.flags.code_coverage result = test.BuildTargetUnitTest( build_target, chroot, packages=packages, blocklist=blocklist, was_built=was_built, code_coverage=code_coverage, testable_packages_optional=testable_packages_optional) if not result.success: # Failed to run tests or some tests failed. # Record all failed packages. for cpv in result.failed_cpvs: package_info_msg = output_proto.failed_packages.add() controller_util.CPVToPackageInfo(cpv, package_info_msg) if result.failed_cpvs: return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE else: return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY sysroot = sysroot_lib.Sysroot(build_target.root) tarball = test.BuildTargetUnitTestTarball(chroot, sysroot, result_path) if tarball: output_proto.tarball_path = tarball deserialize_metrics_log(output_proto.events, prefix=build_target.name)
def testDeserializeGauge(self): """Test deserialization of a gauge.""" response = build_api_test_pb2.TestResultMessage() mock_events = [ MetricEvent(1000, 'a.gauge', OP_GAUGE, arg=17), ] with mock.patch('chromite.api.metrics.metrics.read_metrics_events', return_value=mock_events): metrics.deserialize_metrics_log(response.events) self.assertEqual(len(response.events), 1) self.assertEqual(response.events[0].name, 'a.gauge') self.assertEqual(response.events[0].timestamp_milliseconds, 1000) self.assertEqual(response.events[0].gauge, 17)
def testDeserializeTimer(self): """Test timer math and deserialization into proto objects.""" response = build_api_test_pb2.TestResultMessage() mock_events = [ MetricEvent(600, 'a.b', OP_START_TIMER, arg='100'), MetricEvent(1000, 'a.b', OP_STOP_TIMER, arg='100'), ] with mock.patch('chromite.api.metrics.metrics.read_metrics_events', return_value=mock_events): metrics.deserialize_metrics_log(response.events) self.assertEqual(len(response.events), 1) self.assertEqual(response.events[0].name, 'a.b') self.assertEqual(response.events[0].timestamp_milliseconds, 1000) self.assertEqual(response.events[0].duration_milliseconds, 1000-600)
def testDeserializeNamedEvent(self): """Test deserialization of a named event. This test also includes a prefix to test for proper prepending. """ response = build_api_test_pb2.TestResultMessage() mock_events = [ MetricEvent(1000, 'a.named_event', OP_NAMED_EVENT, arg=None), ] with mock.patch('chromite.api.metrics.metrics.read_metrics_events', return_value=mock_events): metrics.deserialize_metrics_log(response.events, prefix='prefix') self.assertEqual(len(response.events), 1) self.assertEqual(response.events[0].name, 'prefix.a.named_event') self.assertEqual(response.events[0].timestamp_milliseconds, 1000) self.assertFalse(response.events[0].duration_milliseconds)
def InstallPackages(input_proto, output_proto, _config): """Install packages into a sysroot, building as necessary and permitted.""" compile_source = input_proto.flags.compile_source event_file = input_proto.flags.event_file use_goma = input_proto.flags.use_goma target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path) build_target = controller_util.ParseBuildTarget( input_proto.sysroot.build_target) packages = [ controller_util.PackageInfoToString(x) for x in input_proto.packages ] if not target_sysroot.IsToolchainInstalled(): cros_build_lib.Die('Toolchain must first be installed.') _LogBinhost(build_target.name) use_flags = [u.flag for u in input_proto.use_flags] build_packages_config = sysroot.BuildPackagesRunConfig( event_file=event_file, usepkg=not compile_source, install_debug_symbols=True, packages=packages, use_flags=use_flags, use_goma=use_goma) try: sysroot.BuildPackages(build_target, target_sysroot, build_packages_config) except sysroot_lib.PackageInstallError as e: if not e.failed_packages: # No packages to report, so just exit with an error code. return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY # We need to report the failed packages. for package in e.failed_packages: package_info = output_proto.failed_packages.add() controller_util.CPVToPackageInfo(package, package_info) return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE # Read metric events log and pipe them into output_proto.events. deserialize_metrics_log(output_proto.events, prefix=build_target.name)
def BuildTargetUnitTest(input_proto, output_proto, _config): """Run a build target's ebuild unit tests.""" # Required args. board = input_proto.build_target.name result_path = input_proto.result_path # Method flags. # An empty sysroot means build packages was not run. This is used for # certain boards that need to use prebuilts (e.g. grunt's unittest-only). was_built = not input_proto.flags.empty_sysroot # Skipped tests. blacklisted_package_info = input_proto.package_blacklist blacklist = [] for package_info in blacklisted_package_info: blacklist.append(controller_util.PackageInfoToString(package_info)) build_target = build_target_util.BuildTarget(board) chroot = controller_util.ParseChroot(input_proto.chroot) result = test.BuildTargetUnitTest(build_target, chroot, blacklist=blacklist, was_built=was_built) if not result.success: # Failed to run tests or some tests failed. # Record all failed packages. for cpv in result.failed_cpvs: package_info = output_proto.failed_packages.add() controller_util.CPVToPackageInfo(cpv, package_info) if result.failed_cpvs: return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE else: return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY sysroot = sysroot_lib.Sysroot(build_target.root) tarball = test.BuildTargetUnitTestTarball(chroot, sysroot, result_path) if tarball: output_proto.tarball_path = tarball deserialize_metrics_log(output_proto.events, prefix=build_target.name)
def InstallPackages(input_proto, output_proto, _config): """Install packages into a sysroot, building as necessary and permitted.""" compile_source = (input_proto.flags.compile_source or input_proto.flags.toolchain_changed) # Testing if Goma will support unknown compilers now. use_goma = input_proto.flags.use_goma target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path) build_target = controller_util.ParseBuildTarget( input_proto.sysroot.build_target) # Get the package atom for each specified package. The field is optional, so # error only when we cannot parse an atom for each of the given packages. packages = [ controller_util.PackageInfoToCPV(x).cp for x in input_proto.packages ] if input_proto.packages and not all(packages): cros_build_lib.Die( 'Invalid package(s) specified. Unable to parse atom from all packages.' ) package_indexes = [ binpkg.PackageIndexInfo.from_protobuf(x) for x in input_proto.package_indexes ] if not target_sysroot.IsToolchainInstalled(): cros_build_lib.Die('Toolchain must first be installed.') _LogBinhost(build_target.name) use_flags = [u.flag for u in input_proto.use_flags] build_packages_config = sysroot.BuildPackagesRunConfig( usepkg=not compile_source, install_debug_symbols=True, packages=packages, package_indexes=package_indexes, use_flags=use_flags, use_goma=use_goma, incremental_build=False) try: sysroot.BuildPackages(build_target, target_sysroot, build_packages_config) except sysroot_lib.PackageInstallError as e: if not e.failed_packages: # No packages to report, so just exit with an error code. return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY # We need to report the failed packages. for package in e.failed_packages: package_info = output_proto.failed_packages.add() controller_util.CPVToPackageInfo(package, package_info) return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE # Copy goma logs to specified directory if there is a goma_config and # it contains a log_dir to store artifacts. if input_proto.goma_config.log_dir.dir: # Get the goma log directory based on the GLOG_log_dir env variable. # TODO(crbug.com/1045001): Replace environment variable with query to # goma object after goma refactoring allows this. log_source_dir = os.getenv('GLOG_log_dir') if not log_source_dir: cros_build_lib.Die('GLOG_log_dir must be defined.') archiver = goma_lib.LogsArchiver( log_source_dir, dest_dir=input_proto.goma_config.log_dir.dir, stats_file=input_proto.goma_config.stats_file, counterz_file=input_proto.goma_config.counterz_file) archiver_tuple = archiver.Archive() if archiver_tuple.stats_file: output_proto.goma_artifacts.stats_file = archiver_tuple.stats_file if archiver_tuple.counterz_file: output_proto.goma_artifacts.counterz_file = archiver_tuple.counterz_file output_proto.goma_artifacts.log_files[:] = archiver_tuple.log_files # Read metric events log and pipe them into output_proto.events. deserialize_metrics_log(output_proto.events, prefix=build_target.name)