def wrapped_functor(*args, **kwargs):
try:
logging.PrintBuildbotStepName(functor.__name__)
result = functor(*args, **kwargs)
except Exception:
logging.PrintBuildbotStepFailure()
raise
if result:
logging.PrintBuildbotStepFailure()
return result
def PerformStage(self):
# This stage runs only in builders where |android_rev| config is set,
# namely Android PFQ and pre-flight-branch builders.
if not self._android_rev:
logging.info('Not uprevving Android.')
return
android_package = self._run.config.android_package
android_build_branch = self._run.config.android_import_branch
android_version = _GetAndroidVersionFromMetadata(
self._run.attrs.metadata)
android_gts_build_branch = self._run.config.android_gts_build_branch
assert android_package
assert android_build_branch
# |android_version| is usually set by MasterSlaveLKGMSyncStage, but we allow
# it to be unset to indicate uprev'ing to the latest version. In fact, it is
# not set in trybots.
# |android_gts_build_branch| is not set if this builder is not supposed to
# upload GTS bundles.
logging.info('Android package: %s', android_package)
logging.info('Android branch: %s', android_build_branch)
logging.info('Android version: %s', android_version or 'LATEST')
logging.info('Android GTS branch: %s', android_gts_build_branch
or 'N/A')
try:
android_atom_to_build = commands.MarkAndroidAsStable(
buildroot=self._build_root,
tracking_branch=self._run.manifest_branch,
android_package=android_package,
android_build_branch=android_build_branch,
boards=self._boards,
android_version=android_version,
android_gts_build_branch=android_gts_build_branch)
except commands.AndroidIsPinnedUprevError as e:
# If uprev failed due to a pin, record that failure (so that the
# build ultimately fails) but try again without the pin, to allow the
# slave to test the newer version anyway).
android_atom_to_build = e.new_android_atom
results_lib.Results.Record(self.name, e)
logging.PrintBuildbotStepFailure()
logging.error(
'Android is pinned. Unpinning Android and continuing '
'build for Android atom %s. This stage will be marked '
'as failed to prevent an uprev.', android_atom_to_build)
logging.info('Deleting pin file at %s and proceeding.',
ANDROIDPIN_MASK_PATH)
osutils.SafeUnlink(ANDROIDPIN_MASK_PATH)
logging.info('New Android package atom: %s', android_atom_to_build)
if (not android_atom_to_build and self._run.options.buildbot
and self._run.config.build_type == constants.ANDROID_PFQ_TYPE):
logging.info('Android already uprevved. Nothing else to do.')
raise failures_lib.ExitEarlyException(
'UprevAndroidStage finished and exited early.')
def PerformStage(self):
chrome_atom_to_build = None
if self._chrome_rev:
if (self._chrome_rev == constants.CHROME_REV_SPEC and
self._run.options.chrome_version):
self.chrome_version = self._run.options.chrome_version
logging.info('Using chrome version from options.chrome_version: %s',
self.chrome_version)
else:
self.chrome_version = self._GetChromeVersionFromMetadata()
if self.chrome_version:
logging.info('Using chrome version from the metadata dictionary: %s',
self.chrome_version)
# Perform chrome uprev.
try:
chrome_atom_to_build = commands.MarkChromeAsStable(
self._build_root, self._run.manifest_branch,
self._chrome_rev, self._boards,
chrome_version=self.chrome_version)
except commands.ChromeIsPinnedUprevError as e:
# If uprev failed due to a chrome pin, record that failure (so that the
# build ultimately fails) but try again without the pin, to allow the
# slave to test the newer chrome anyway).
chrome_atom_to_build = e.new_chrome_atom
if chrome_atom_to_build:
results_lib.Results.Record(self.name, e)
logging.PrintBuildbotStepFailure()
logging.error('Chrome is pinned. Attempting to continue build for '
'chrome atom %s anyway but build will ultimately fail.',
chrome_atom_to_build)
logging.info('Deleting pin file at %s and proceeding.',
CHROMEPIN_MASK_PATH)
osutils.SafeUnlink(CHROMEPIN_MASK_PATH)
else:
raise
kwargs = {}
if self._chrome_rev == constants.CHROME_REV_SPEC:
kwargs['revision'] = self.chrome_version
logging.PrintBuildbotStepText('revision %s' % kwargs['revision'])
else:
if not self.chrome_version:
self.chrome_version = self._run.DetermineChromeVersion()
kwargs['tag'] = self.chrome_version
logging.PrintBuildbotStepText('tag %s' % kwargs['tag'])
useflags = self._run.config.useflags
commands.SyncChrome(self._build_root, self._run.options.chrome_root,
useflags, **kwargs)
if (self._chrome_rev and not chrome_atom_to_build and
self._run.options.buildbot and
self._run.config.build_type == constants.CHROME_PFQ_TYPE):
logging.info('Chrome already uprevved. Nothing else to do.')
raise failures_lib.ExitEarlyException(
'SyncChromeStage finished and exited early.')
def PerformStage(self):
chrome_atom_to_build = None
if self._chrome_rev:
if (self._chrome_rev == constants.CHROME_REV_SPEC and
self._run.options.chrome_version):
self.chrome_version = self._run.options.chrome_version
logging.info('Using chrome version from options.chrome_version: %s',
self.chrome_version)
else:
self.chrome_version = self._GetChromeVersionFromMetadata()
if self.chrome_version:
logging.info('Using chrome version from the metadata dictionary: %s',
self.chrome_version)
# Perform chrome uprev.
try:
chrome_atom_to_build = commands.MarkChromeAsStable(
self._build_root, self._run.manifest_branch,
self._chrome_rev, self._boards,
chrome_version=self.chrome_version)
except commands.ChromeIsPinnedUprevError as e:
# If uprev failed due to a chrome pin, record that failure (so that the
# build ultimately fails) but try again without the pin, to allow the
# slave to test the newer chrome anyway).
chrome_atom_to_build = e.new_chrome_atom
if chrome_atom_to_build:
results_lib.Results.Record(self.name, e)
logging.PrintBuildbotStepFailure()
logging.error('Chrome is pinned. Unpinning chrome and continuing '
'build for chrome atom %s. This stage will be marked '
'as failed to prevent an uprev.',
chrome_atom_to_build)
logging.info('Deleting pin file at %s and proceeding.',
CHROMEPIN_MASK_PATH)
osutils.SafeUnlink(CHROMEPIN_MASK_PATH)
else:
raise
kwargs = {}
if self._chrome_rev == constants.CHROME_REV_SPEC:
kwargs['revision'] = self.chrome_version
logging.PrintBuildbotStepText('revision %s' % kwargs['revision'])
else:
if not self.chrome_version:
self.chrome_version = self._run.DetermineChromeVersion()
kwargs['tag'] = self.chrome_version
logging.PrintBuildbotStepText('tag %s' % kwargs['tag'])
useflags = self._run.config.useflags
git_cache_dir = (
self._run.options.chrome_preload_dir or self._run.options.git_cache_dir)
commands.SyncChrome(self._build_root,
self._run.options.chrome_root,
useflags,
git_cache_dir=git_cache_dir,
**kwargs)
def run(self):
"""Main function for tee subprocess."""
failed = True
input_fd = None
try:
signal.signal(signal.SIGINT, _TeeProcessSignalHandler)
signal.signal(signal.SIGTERM, _TeeProcessSignalHandler)
# Cleanup every fd except for what we use.
self._CloseUnnecessaryFds()
# Read from the pipe.
input_fd = self._reader_pipe
# Create list of files to write to.
# Not passing 3 argument (0) for unbuffered output because this is not
# supported in Python 3 and there are issues in Python 2 -- see
# https://bugs.python.org/issue17404.
output_files = [os.fdopen(sys.stdout.fileno(), 'w')]
for filename in self._output_filenames:
output_files.append(open(filename, 'w'))
# Send all data from the one input to all the outputs.
_tee(input_fd, output_files, self._complain)
failed = False
except ToldToDie:
failed = False
except Exception as e:
tb = traceback.format_exc()
logging.PrintBuildbotStepFailure(self._error_handle)
self._error_handle.write(
'Unhandled exception occured in tee:\n%s\n' % (tb, ))
# Try to signal the parent telling them of our
# imminent demise.
finally:
# Close input.
if input_fd:
os.close(input_fd)
if failed:
try:
os.kill(self.master_pid, signal.SIGTERM)
except Exception as e:
self._error_handle.write('\nTee failed signaling %s\n' % e)
# Finally, kill ourself.
# Specifically do it in a fashion that ensures no inherited
# cleanup code from our parent process is ran- leave that to
# the parent.
# pylint: disable=protected-access
os._exit(0)
def _HandleExceptionAsError(cls, exc_info):
"""Handle an exception as an error, but ignore stage retry settings.
Meant as a helper for _HandleStageException code only.
Args:
exc_info: A (type, value, traceback) tuple as returned by sys.exc_info().
Returns:
Result tuple of (exception, description, retrying).
"""
# Tell the user about the exception, and record it.
retrying = False
description = cls._StringifyException(exc_info)
logging.PrintBuildbotStepFailure()
logging.error(description)
return (exc_info[1], description, retrying)
def kill_us(sig_num, frame):
# While this SystemExit *should* crash it's way back up the
# stack to our exit handler, we do have live/production code
# that uses blanket except statements which could suppress this.
# As such, keep scheduling alarms until our exit handler runs.
# Note that there is a potential conflict via this code, and
# RunCommand's kill_timeout; thus we set the alarming interval
# fairly high.
signal.alarm(60)
# The cbuildbot stage that gets aborted by this timeout should be treated as
# failed by buildbot.
error_message = ("Timeout occurred- waited %i seconds, failing." %
max_run_time)
if display_message:
error_message += ' Timeout reason: %s' % display_message
logging.PrintBuildbotStepFailure()
logging.error(error_message)
raise SystemExit(error_message)
def PerformStage(self):
android_atom_to_build = None
if self._android_rev:
self.android_version = self._GetAndroidVersionFromMetadata()
if self.android_version:
logging.info(
'Using Android version from the metadata dictionary: %s',
self.android_version)
try:
android_atom_to_build = commands.MarkAndroidAsStable(
self._build_root,
self._run.manifest_branch,
self._boards,
android_version=self.android_version)
except commands.AndroidIsPinnedUprevError as e:
# If uprev failed due to a pin, record that failure (so that the
# build ultimately fails) but try again without the pin, to allow the
# slave to test the newer version anyway).
android_atom_to_build = e.new_android_atom
if android_atom_to_build:
results_lib.Results.Record(self.name, e)
logging.PrintBuildbotStepFailure()
logging.error(
'Android is pinned. Attempting to continue build for '
'Android atom %s anyway but build will ultimately '
'fail.', android_atom_to_build)
logging.info('Deleting pin file at %s and proceeding.',
ANDROIDPIN_MASK_PATH)
osutils.SafeUnlink(ANDROIDPIN_MASK_PATH)
else:
raise
if (self._android_rev and not android_atom_to_build
and self._run.options.buildbot
and self._run.config.build_type == constants.ANDROID_PFQ_TYPE):
logging.info('Android already uprevved. Nothing else to do.')
raise failures_lib.ExitEarlyException(
'UprevAndroidStage finished and exited early.')
def Wait(self):
"""Wait for the task to complete.
Output from the task is printed as it runs.
If an exception occurs, return a string containing the traceback.
"""
try:
# Flush stdout and stderr to be sure no output is interleaved.
sys.stdout.flush()
sys.stderr.flush()
# File position pointers are shared across processes, so we must open
# our own file descriptor to ensure output is not lost.
self._WaitForStartup()
silent_death_time = time.time() + self.SILENT_TIMEOUT
results = []
with open(self._output.name, 'r') as output:
pos = 0
running, exited_cleanly, task_errors, all_errors = (True, False, [], [])
while running:
# Check whether the process is still alive.
running = self.is_alive()
try:
errors, results = \
self._queue.get(True, self.PRINT_INTERVAL)
if errors:
task_errors.extend(errors)
all_errors.extend(errors)
running = False
exited_cleanly = True
except Queue.Empty:
pass
if not running:
# Wait for the process to actually exit. If the child doesn't exit
# in a timely fashion, kill it.
self.join(self.EXIT_TIMEOUT)
if self.exitcode is None:
msg = '%r hung for %r seconds' % (self, self.EXIT_TIMEOUT)
all_errors.extend(
failures_lib.CreateExceptInfo(ProcessExitTimeout(msg), ''))
self._KillChildren([self])
elif not exited_cleanly:
msg = ('%r exited unexpectedly with code %s'
% (self, self.exitcode))
all_errors.extend(
failures_lib.CreateExceptInfo(ProcessUnexpectedExit(msg), ''))
# Read output from process.
output.seek(pos)
buf = output.read(_BUFSIZE)
if len(buf) > 0:
silent_death_time = time.time() + self.SILENT_TIMEOUT
elif running and time.time() > silent_death_time:
msg = ('No output from %r for %r seconds' %
(self, self.SILENT_TIMEOUT))
all_errors.extend(
failures_lib.CreateExceptInfo(ProcessSilentTimeout(msg), ''))
self._KillChildren([self])
# Read remaining output from the process.
output.seek(pos)
buf = output.read(_BUFSIZE)
running = False
# Print output so far.
while len(buf) > 0:
sys.stdout.write(buf)
pos += len(buf)
if len(buf) < _BUFSIZE:
break
buf = output.read(_BUFSIZE)
# Print error messages if anything exceptional occurred.
if len(all_errors) > len(task_errors):
logging.PrintBuildbotStepFailure()
msg = '\n'.join(x.str for x in all_errors if x)
logging.warning(msg)
traceback.print_stack()
sys.stdout.flush()
sys.stderr.flush()
# Propagate any results.
for result in results:
results_lib.Results.Record(*result)
finally:
self.Cleanup(silent=True)
# If an error occurred, return it.
return all_errors
