def RunBuildStages(self):
"""Runs through the stages to perform the build and resulting tests."""
# Prepare stages to run in background. If child_configs exist then
# run each of those here, otherwise use default config.
builder_runs = self._run.GetUngroupedBuilderRuns()
tasks = []
for builder_run in builder_runs:
# Prepare a local archive directory for each "run".
builder_run.GetArchive().SetupArchivePath()
for board in builder_run.config.boards:
archive_stage = self._GetStageInstance(
artifact_stages.ArchiveStage, board, builder_run=builder_run,
chrome_version=self._run.attrs.chrome_version)
board_config = BoardConfig(board, builder_run.config.name)
self.archive_stages[board_config] = archive_stage
tasks.append((builder_run, board))
# Set up a process pool to run test/archive stages in the background.
# This process runs task(board) for each board added to the queue.
task_runner = self._RunBackgroundStagesForBoardAndMarkAsSuccessful
with parallel.BackgroundTaskRunner(task_runner) as queue:
for builder_run, board in tasks:
if not builder_run.config.build_packages_in_background:
# Run BuildPackages in the foreground, generating or using AFDO data
# if requested.
kwargs = {'builder_run': builder_run}
if builder_run.config.afdo_generate_min:
kwargs['afdo_generate_min'] = True
elif builder_run.config.afdo_use:
kwargs['afdo_use'] = True
self._RunStage(build_stages.BuildPackagesStage, board,
update_metadata=True, **kwargs)
if (builder_run.config.afdo_generate_min and
afdo.CanGenerateAFDOData(board)):
# Generate the AFDO data before allowing any other tasks to run.
self._RunStage(build_stages.BuildImageStage, board, **kwargs)
self._RunStage(artifact_stages.UploadTestArtifactsStage, board,
builder_run=builder_run,
suffix='[afdo_generate_min]')
for suite in builder_run.config.hw_tests:
self._RunStage(test_stages.HWTestStage, board, suite,
builder_run=builder_run)
self._RunStage(afdo_stages.AFDODataGenerateStage, board,
builder_run=builder_run)
if (builder_run.config.afdo_generate_min and
builder_run.config.afdo_update_ebuild):
self._RunStage(afdo_stages.AFDOUpdateEbuildStage,
builder_run=builder_run)
# Kick off our background stages.
queue.put([builder_run, board])
def PerformStage(self):
"""Collect a 'perf' profile and convert it into the AFDO format."""
super(AFDODataGenerateStage, self).PerformStage()
board = self._current_board
if not afdo.CanGenerateAFDOData(board):
logging.warning('Board %s cannot generate its own AFDO profile.',
board)
return
arch = self._GetCurrentArch()
buildroot = self._build_root
gs_context = gs.GSContext()
cpv = portage_util.BestVisible(constants.CHROME_CP,
buildroot=buildroot)
afdo_file = None
# Generation of AFDO could fail for different reasons.
# We will ignore the failures and let the master PFQ builder try
# to find an older AFDO profile.
try:
if afdo.WaitForAFDOPerfData(cpv, arch, buildroot, gs_context):
afdo_file = afdo.GenerateAFDOData(cpv, arch, board, buildroot,
gs_context)
assert afdo_file
logging.info('Generated %s AFDO profile %s', arch, afdo_file)
else:
raise afdo.MissingAFDOData(
'Could not find current "perf" profile. '
'Master PFQ builder will try to use stale '
'AFDO profile.')
# Will let system-exiting exceptions through.
except Exception:
logging.PrintBuildbotStepWarnings()
logging.warning('AFDO profile generation failed with exception ',
exc_info=True)
alert_msg = ('Please triage. This will become a fatal error.\n\n'
'arch=%s buildroot=%s\n\nURL=%s' %
(arch, buildroot, self._run.ConstructDashboardURL()))
subject_msg = (
'Failure in generation of AFDO Data for builder %s' %
self._run.config.name)
alerts.SendEmailLog(subject_msg,
afdo.AFDO_ALERT_RECIPIENTS,
server=alerts.SmtpServer(
constants.GOLO_SMTP_SERVER),
message=alert_msg)
# Re-raise whatever exception we got here. This stage will only
# generate a warning but we want to make sure the warning is
# generated.
raise
def PerformStage(self):
"""Collect a 'perf' profile and convert it into the AFDO format."""
super(AFDODataGenerateStage, self).PerformStage()
board = self._current_board
if not afdo.CanGenerateAFDOData(board):
logging.warning('Board %s cannot generate its own AFDO profile.',
board)
return
arch = self._GetCurrentArch()
buildroot = self._build_root
gs_context = gs.GSContext()
cpv = portage_util.PortageqBestVisible(constants.CHROME_CP,
cwd=buildroot)
afdo_file = None
# We have a mismatch between how we version the perf.data we collect and
# how we version our AFDO profiles.
#
# This mismatch can cause us to generate garbage profiles, so we skip
# profile updates for non-r1 revisions of Chrome.
#
# Going into more detail, a perf.data file looks like:
# chromeos-chrome-amd64-68.0.3440.9.perf.data.bz2
#
# An AFDO profile looks like:
# chromeos-chrome-amd64-68.0.3440.9_rc-r1.afdo.bz2
#
# And an unstripped Chrome looks like:
# chromeos-chrome-amd64-68.0.3440.9_rc-r1.debug.bz2
#
# Notably, the perf.data is lacking the revision number of the Chrome it
# was gathered on. This is problematic, since if there's a rev bump, we'll
# end up using the perf.data collected on Chrome version $N-r1 with a
# Chrome binary built from Chrome version $N-r2, which may have an entirely
# different layout than Chrome version $N-r1.
if cpv.rev != 'r1':
logging.warning(
'Non-r1 version of Chrome detected; skipping AFDO generation')
return
# Generation of AFDO could fail for different reasons.
# We will ignore the failures and let the master PFQ builder try
# to find an older AFDO profile.
try:
if afdo.WaitForAFDOPerfData(cpv, arch, buildroot, gs_context):
afdo_file, uploaded_afdo = afdo.GenerateAFDOData(
cpv, arch, board, buildroot, gs_context)
assert afdo_file
logging.info('Generated %s AFDO profile %s', arch, afdo_file)
# If there's no new profile, merging would only be redoing the last
# merge and uploading nothing.
if not uploaded_afdo:
logging.info('AFDO profile already existed in GS. Quit')
return
merged_file, uploaded_merged = \
afdo.CreateAndUploadMergedAFDOProfile(gs_context, buildroot,
afdo_file)
if merged_file is not None:
logging.info('Generated %s merged AFDO profile %s', arch,
merged_file)
# TODO(gbiv): once there's clarity that merged profiles are working
# (e.g. a week goes by with Android/Linux mostly-happily using them),
# we may want to turn them on for CrOS. Until then, `latest` is always
# the raw AFDO file.
if uploaded_merged and False:
newest_afdo_file = merged_file
else:
newest_afdo_file = afdo_file
afdo.UpdateLatestAFDOProfileInGS(cpv, arch, buildroot,
newest_afdo_file, gs_context)
logging.info('Pointed newest profile at %s', newest_afdo_file)
else:
raise afdo.MissingAFDOData(
'Could not find current "perf" profile. '
'Master PFQ builder will try to use stale '
'AFDO profile.')
# Will let system-exiting exceptions through.
except Exception:
logging.PrintBuildbotStepWarnings()
logging.warning('AFDO profile generation failed with exception ',
exc_info=True)
alert_msg = ('Please triage. This will become a fatal error.\n\n'
'arch=%s buildroot=%s\n\nURL=%s' %
(arch, buildroot, self._run.ConstructDashboardURL()))
subject_msg = (
'Failure in generation of AFDO Data for builder %s' %
self._run.config.name)
alerts.SendEmailLog(subject_msg,
afdo.AFDO_ALERT_RECIPIENTS,
server=alerts.SmtpServer(
constants.GOLO_SMTP_SERVER),
message=alert_msg)
# Re-raise whatever exception we got here. This stage will only
# generate a warning but we want to make sure the warning is
# generated.
raise
def RunBuildStages(self):
"""Runs through the stages to perform the build and resulting tests."""
# Prepare stages to run in background. If child_configs exist then
# run each of those here, otherwise use default config.
builder_runs = self._run.GetUngroupedBuilderRuns()
tasks = []
for builder_run in builder_runs:
# Prepare a local archive directory for each "run".
builder_run.GetArchive().SetupArchivePath()
for board in builder_run.config.boards:
archive_stage = self._GetStageInstance(
artifact_stages.ArchiveStage,
board,
builder_run=builder_run,
chrome_version=self._run.attrs.chrome_version)
board_config = BoardConfig(board, builder_run.config.name)
self.archive_stages[board_config] = archive_stage
tasks.append((builder_run, board))
# Set up a process pool to run test/archive stages in the background.
# This process runs task(board) for each board added to the queue.
task_runner = self._RunBackgroundStagesForBoardAndMarkAsSuccessful
with parallel.BackgroundTaskRunner(task_runner) as queue:
for builder_run, board in tasks:
# Skip generate benchmark AFDO if the board is not suitable or
# if it's already in the bucket
if builder_run.config.afdo_generate_async and \
toolchain_util.CanGenerateAFDOData(board) and \
toolchain_util.CheckAFDOArtifactExists(
buildroot=builder_run.buildroot,
chrome_root=builder_run.options.chrome_root,
board=board,
target='benchmark_afdo'):
continue
# Only generate orderfile if Chrome is upreved since last generation
if builder_run.config.orderfile_generate and \
toolchain_util.CheckAFDOArtifactExists(
buildroot=builder_run.buildroot,
chrome_root=builder_run.options.chrome_root,
board=board,
target='orderfile_generate'):
continue
# Update Chrome ebuild with unvetted orderfile
if builder_run.config.orderfile_verify:
# Skip verifying orderfile if it's already verified.
if toolchain_util.CheckAFDOArtifactExists(
buildroot=builder_run.buildroot,
chrome_root=builder_run.options.chrome_root,
board=board,
target='orderfile_verify'):
continue
self._RunStage(
afdo_stages.OrderfileUpdateChromeEbuildStage,
board,
builder_run=builder_run)
if builder_run.config.kernel_afdo_verify:
# Skip verifying kernel AFDO if it's already verified.
if toolchain_util.CheckAFDOArtifactExists(
buildroot=builder_run.buildroot,
chrome_root=builder_run.options.chrome_root,
board=board,
target='kernel_afdo'):
continue
self._RunStage(afdo_stages.KernelAFDOUpdateEbuildStage,
board,
builder_run=builder_run)
if builder_run.config.chrome_afdo_verify:
# Skip verifying Chrome AFDO if both benchmark and CWP are verified.
if toolchain_util.CheckAFDOArtifactExists(
buildroot=builder_run.buildroot,
chrome_root=builder_run.options.chrome_root,
board=board,
target='chrome_afdo'):
continue
self._RunStage(afdo_stages.ChromeAFDOUpdateEbuildStage,
board,
builder_run=builder_run)
# Run BuildPackages in the foreground, generating or using AFDO data
# if requested.
kwargs = {'builder_run': builder_run}
if builder_run.config.afdo_generate_min:
kwargs['afdo_generate_min'] = True
elif builder_run.config.afdo_use:
kwargs['afdo_use'] = True
self._RunStage(build_stages.BuildPackagesStage,
board,
update_metadata=True,
**kwargs)
if (builder_run.config.afdo_generate_min
and afdo.CanGenerateAFDOData(board)):
# Generate the AFDO data before allowing any other tasks to run.
self._RunStage(build_stages.BuildImageStage, board,
**kwargs)
self._RunStage(artifact_stages.UploadTestArtifactsStage,
board,
builder_run=builder_run,
suffix='[afdo_generate_min]')
for suite in builder_run.config.hw_tests:
self._RunStage(test_stages.SkylabHWTestStage,
board,
suite,
builder_run=builder_run)
self._RunStage(afdo_stages.AFDODataGenerateStage,
board,
builder_run=builder_run)
if (builder_run.config.afdo_generate_min
and builder_run.config.afdo_update_ebuild):
self._RunStage(afdo_stages.AFDOUpdateChromeEbuildStage,
builder_run=builder_run)
self._RunStage(afdo_stages.AFDOUpdateKernelEbuildStage,
builder_run=builder_run)
# Kick off our background stages.
queue.put([builder_run, board])
