def RunBuildStages(self): """Runs through the stages to perform the build and resulting tests.""" # Prepare stages to run in background. If child_configs exist then # run each of those here, otherwise use default config. builder_runs = self._run.GetUngroupedBuilderRuns() tasks = [] for builder_run in builder_runs: # Prepare a local archive directory for each "run". builder_run.GetArchive().SetupArchivePath() for board in builder_run.config.boards: archive_stage = self._GetStageInstance( artifact_stages.ArchiveStage, board, builder_run=builder_run, chrome_version=self._run.attrs.chrome_version) board_config = BoardConfig(board, builder_run.config.name) self.archive_stages[board_config] = archive_stage tasks.append((builder_run, board)) # Set up a process pool to run test/archive stages in the background. # This process runs task(board) for each board added to the queue. task_runner = self._RunBackgroundStagesForBoardAndMarkAsSuccessful with parallel.BackgroundTaskRunner(task_runner) as queue: for builder_run, board in tasks: if not builder_run.config.build_packages_in_background: # Run BuildPackages in the foreground, generating or using AFDO data # if requested. kwargs = {'builder_run': builder_run} if builder_run.config.afdo_generate_min: kwargs['afdo_generate_min'] = True elif builder_run.config.afdo_use: kwargs['afdo_use'] = True self._RunStage(build_stages.BuildPackagesStage, board, update_metadata=True, **kwargs) if (builder_run.config.afdo_generate_min and afdo.CanGenerateAFDOData(board)): # Generate the AFDO data before allowing any other tasks to run. self._RunStage(build_stages.BuildImageStage, board, **kwargs) self._RunStage(artifact_stages.UploadTestArtifactsStage, board, builder_run=builder_run, suffix='[afdo_generate_min]') for suite in builder_run.config.hw_tests: self._RunStage(test_stages.HWTestStage, board, suite, builder_run=builder_run) self._RunStage(afdo_stages.AFDODataGenerateStage, board, builder_run=builder_run) if (builder_run.config.afdo_generate_min and builder_run.config.afdo_update_ebuild): self._RunStage(afdo_stages.AFDOUpdateEbuildStage, builder_run=builder_run) # Kick off our background stages. queue.put([builder_run, board])
def PerformStage(self): """Collect a 'perf' profile and convert it into the AFDO format.""" super(AFDODataGenerateStage, self).PerformStage() board = self._current_board if not afdo.CanGenerateAFDOData(board): logging.warning('Board %s cannot generate its own AFDO profile.', board) return arch = self._GetCurrentArch() buildroot = self._build_root gs_context = gs.GSContext() cpv = portage_util.BestVisible(constants.CHROME_CP, buildroot=buildroot) afdo_file = None # Generation of AFDO could fail for different reasons. # We will ignore the failures and let the master PFQ builder try # to find an older AFDO profile. try: if afdo.WaitForAFDOPerfData(cpv, arch, buildroot, gs_context): afdo_file = afdo.GenerateAFDOData(cpv, arch, board, buildroot, gs_context) assert afdo_file logging.info('Generated %s AFDO profile %s', arch, afdo_file) else: raise afdo.MissingAFDOData( 'Could not find current "perf" profile. ' 'Master PFQ builder will try to use stale ' 'AFDO profile.') # Will let system-exiting exceptions through. except Exception: logging.PrintBuildbotStepWarnings() logging.warning('AFDO profile generation failed with exception ', exc_info=True) alert_msg = ('Please triage. This will become a fatal error.\n\n' 'arch=%s buildroot=%s\n\nURL=%s' % (arch, buildroot, self._run.ConstructDashboardURL())) subject_msg = ( 'Failure in generation of AFDO Data for builder %s' % self._run.config.name) alerts.SendEmailLog(subject_msg, afdo.AFDO_ALERT_RECIPIENTS, server=alerts.SmtpServer( constants.GOLO_SMTP_SERVER), message=alert_msg) # Re-raise whatever exception we got here. This stage will only # generate a warning but we want to make sure the warning is # generated. raise
def PerformStage(self): """Collect a 'perf' profile and convert it into the AFDO format.""" super(AFDODataGenerateStage, self).PerformStage() board = self._current_board if not afdo.CanGenerateAFDOData(board): logging.warning('Board %s cannot generate its own AFDO profile.', board) return arch = self._GetCurrentArch() buildroot = self._build_root gs_context = gs.GSContext() cpv = portage_util.PortageqBestVisible(constants.CHROME_CP, cwd=buildroot) afdo_file = None # We have a mismatch between how we version the perf.data we collect and # how we version our AFDO profiles. # # This mismatch can cause us to generate garbage profiles, so we skip # profile updates for non-r1 revisions of Chrome. # # Going into more detail, a perf.data file looks like: # chromeos-chrome-amd64-68.0.3440.9.perf.data.bz2 # # An AFDO profile looks like: # chromeos-chrome-amd64-68.0.3440.9_rc-r1.afdo.bz2 # # And an unstripped Chrome looks like: # chromeos-chrome-amd64-68.0.3440.9_rc-r1.debug.bz2 # # Notably, the perf.data is lacking the revision number of the Chrome it # was gathered on. This is problematic, since if there's a rev bump, we'll # end up using the perf.data collected on Chrome version $N-r1 with a # Chrome binary built from Chrome version $N-r2, which may have an entirely # different layout than Chrome version $N-r1. if cpv.rev != 'r1': logging.warning( 'Non-r1 version of Chrome detected; skipping AFDO generation') return # Generation of AFDO could fail for different reasons. # We will ignore the failures and let the master PFQ builder try # to find an older AFDO profile. try: if afdo.WaitForAFDOPerfData(cpv, arch, buildroot, gs_context): afdo_file, uploaded_afdo = afdo.GenerateAFDOData( cpv, arch, board, buildroot, gs_context) assert afdo_file logging.info('Generated %s AFDO profile %s', arch, afdo_file) # If there's no new profile, merging would only be redoing the last # merge and uploading nothing. if not uploaded_afdo: logging.info('AFDO profile already existed in GS. Quit') return merged_file, uploaded_merged = \ afdo.CreateAndUploadMergedAFDOProfile(gs_context, buildroot, afdo_file) if merged_file is not None: logging.info('Generated %s merged AFDO profile %s', arch, merged_file) # TODO(gbiv): once there's clarity that merged profiles are working # (e.g. a week goes by with Android/Linux mostly-happily using them), # we may want to turn them on for CrOS. Until then, `latest` is always # the raw AFDO file. if uploaded_merged and False: newest_afdo_file = merged_file else: newest_afdo_file = afdo_file afdo.UpdateLatestAFDOProfileInGS(cpv, arch, buildroot, newest_afdo_file, gs_context) logging.info('Pointed newest profile at %s', newest_afdo_file) else: raise afdo.MissingAFDOData( 'Could not find current "perf" profile. ' 'Master PFQ builder will try to use stale ' 'AFDO profile.') # Will let system-exiting exceptions through. except Exception: logging.PrintBuildbotStepWarnings() logging.warning('AFDO profile generation failed with exception ', exc_info=True) alert_msg = ('Please triage. This will become a fatal error.\n\n' 'arch=%s buildroot=%s\n\nURL=%s' % (arch, buildroot, self._run.ConstructDashboardURL())) subject_msg = ( 'Failure in generation of AFDO Data for builder %s' % self._run.config.name) alerts.SendEmailLog(subject_msg, afdo.AFDO_ALERT_RECIPIENTS, server=alerts.SmtpServer( constants.GOLO_SMTP_SERVER), message=alert_msg) # Re-raise whatever exception we got here. This stage will only # generate a warning but we want to make sure the warning is # generated. raise
def RunBuildStages(self): """Runs through the stages to perform the build and resulting tests.""" # Prepare stages to run in background. If child_configs exist then # run each of those here, otherwise use default config. builder_runs = self._run.GetUngroupedBuilderRuns() tasks = [] for builder_run in builder_runs: # Prepare a local archive directory for each "run". builder_run.GetArchive().SetupArchivePath() for board in builder_run.config.boards: archive_stage = self._GetStageInstance( artifact_stages.ArchiveStage, board, builder_run=builder_run, chrome_version=self._run.attrs.chrome_version) board_config = BoardConfig(board, builder_run.config.name) self.archive_stages[board_config] = archive_stage tasks.append((builder_run, board)) # Set up a process pool to run test/archive stages in the background. # This process runs task(board) for each board added to the queue. task_runner = self._RunBackgroundStagesForBoardAndMarkAsSuccessful with parallel.BackgroundTaskRunner(task_runner) as queue: for builder_run, board in tasks: # Skip generate benchmark AFDO if the board is not suitable or # if it's already in the bucket if builder_run.config.afdo_generate_async and \ toolchain_util.CanGenerateAFDOData(board) and \ toolchain_util.CheckAFDOArtifactExists( buildroot=builder_run.buildroot, chrome_root=builder_run.options.chrome_root, board=board, target='benchmark_afdo'): continue # Only generate orderfile if Chrome is upreved since last generation if builder_run.config.orderfile_generate and \ toolchain_util.CheckAFDOArtifactExists( buildroot=builder_run.buildroot, chrome_root=builder_run.options.chrome_root, board=board, target='orderfile_generate'): continue # Update Chrome ebuild with unvetted orderfile if builder_run.config.orderfile_verify: # Skip verifying orderfile if it's already verified. if toolchain_util.CheckAFDOArtifactExists( buildroot=builder_run.buildroot, chrome_root=builder_run.options.chrome_root, board=board, target='orderfile_verify'): continue self._RunStage( afdo_stages.OrderfileUpdateChromeEbuildStage, board, builder_run=builder_run) if builder_run.config.kernel_afdo_verify: # Skip verifying kernel AFDO if it's already verified. if toolchain_util.CheckAFDOArtifactExists( buildroot=builder_run.buildroot, chrome_root=builder_run.options.chrome_root, board=board, target='kernel_afdo'): continue self._RunStage(afdo_stages.KernelAFDOUpdateEbuildStage, board, builder_run=builder_run) if builder_run.config.chrome_afdo_verify: # Skip verifying Chrome AFDO if both benchmark and CWP are verified. if toolchain_util.CheckAFDOArtifactExists( buildroot=builder_run.buildroot, chrome_root=builder_run.options.chrome_root, board=board, target='chrome_afdo'): continue self._RunStage(afdo_stages.ChromeAFDOUpdateEbuildStage, board, builder_run=builder_run) # Run BuildPackages in the foreground, generating or using AFDO data # if requested. kwargs = {'builder_run': builder_run} if builder_run.config.afdo_generate_min: kwargs['afdo_generate_min'] = True elif builder_run.config.afdo_use: kwargs['afdo_use'] = True self._RunStage(build_stages.BuildPackagesStage, board, update_metadata=True, **kwargs) if (builder_run.config.afdo_generate_min and afdo.CanGenerateAFDOData(board)): # Generate the AFDO data before allowing any other tasks to run. self._RunStage(build_stages.BuildImageStage, board, **kwargs) self._RunStage(artifact_stages.UploadTestArtifactsStage, board, builder_run=builder_run, suffix='[afdo_generate_min]') for suite in builder_run.config.hw_tests: self._RunStage(test_stages.SkylabHWTestStage, board, suite, builder_run=builder_run) self._RunStage(afdo_stages.AFDODataGenerateStage, board, builder_run=builder_run) if (builder_run.config.afdo_generate_min and builder_run.config.afdo_update_ebuild): self._RunStage(afdo_stages.AFDOUpdateChromeEbuildStage, builder_run=builder_run) self._RunStage(afdo_stages.AFDOUpdateKernelEbuildStage, builder_run=builder_run) # Kick off our background stages. queue.put([builder_run, board])