def wrapped_functor(*args, **kwargs): try: logging.PrintBuildbotStepName(functor.__name__) result = functor(*args, **kwargs) except Exception: logging.PrintBuildbotStepFailure() raise if result: logging.PrintBuildbotStepFailure() return result
def PerformStage(self): # This stage runs only in builders where |android_rev| config is set, # namely Android PFQ and pre-flight-branch builders. if not self._android_rev: logging.info('Not uprevving Android.') return android_package = self._run.config.android_package android_build_branch = self._run.config.android_import_branch android_version = _GetAndroidVersionFromMetadata( self._run.attrs.metadata) android_gts_build_branch = self._run.config.android_gts_build_branch assert android_package assert android_build_branch # |android_version| is usually set by MasterSlaveLKGMSyncStage, but we allow # it to be unset to indicate uprev'ing to the latest version. In fact, it is # not set in trybots. # |android_gts_build_branch| is not set if this builder is not supposed to # upload GTS bundles. logging.info('Android package: %s', android_package) logging.info('Android branch: %s', android_build_branch) logging.info('Android version: %s', android_version or 'LATEST') logging.info('Android GTS branch: %s', android_gts_build_branch or 'N/A') try: android_atom_to_build = commands.MarkAndroidAsStable( buildroot=self._build_root, tracking_branch=self._run.manifest_branch, android_package=android_package, android_build_branch=android_build_branch, boards=self._boards, android_version=android_version, android_gts_build_branch=android_gts_build_branch) except commands.AndroidIsPinnedUprevError as e: # If uprev failed due to a pin, record that failure (so that the # build ultimately fails) but try again without the pin, to allow the # slave to test the newer version anyway). android_atom_to_build = e.new_android_atom results_lib.Results.Record(self.name, e) logging.PrintBuildbotStepFailure() logging.error( 'Android is pinned. Unpinning Android and continuing ' 'build for Android atom %s. This stage will be marked ' 'as failed to prevent an uprev.', android_atom_to_build) logging.info('Deleting pin file at %s and proceeding.', ANDROIDPIN_MASK_PATH) osutils.SafeUnlink(ANDROIDPIN_MASK_PATH) logging.info('New Android package atom: %s', android_atom_to_build) if (not android_atom_to_build and self._run.options.buildbot and self._run.config.build_type == constants.ANDROID_PFQ_TYPE): logging.info('Android already uprevved. Nothing else to do.') raise failures_lib.ExitEarlyException( 'UprevAndroidStage finished and exited early.')
def PerformStage(self): chrome_atom_to_build = None if self._chrome_rev: if (self._chrome_rev == constants.CHROME_REV_SPEC and self._run.options.chrome_version): self.chrome_version = self._run.options.chrome_version logging.info('Using chrome version from options.chrome_version: %s', self.chrome_version) else: self.chrome_version = self._GetChromeVersionFromMetadata() if self.chrome_version: logging.info('Using chrome version from the metadata dictionary: %s', self.chrome_version) # Perform chrome uprev. try: chrome_atom_to_build = commands.MarkChromeAsStable( self._build_root, self._run.manifest_branch, self._chrome_rev, self._boards, chrome_version=self.chrome_version) except commands.ChromeIsPinnedUprevError as e: # If uprev failed due to a chrome pin, record that failure (so that the # build ultimately fails) but try again without the pin, to allow the # slave to test the newer chrome anyway). chrome_atom_to_build = e.new_chrome_atom if chrome_atom_to_build: results_lib.Results.Record(self.name, e) logging.PrintBuildbotStepFailure() logging.error('Chrome is pinned. Attempting to continue build for ' 'chrome atom %s anyway but build will ultimately fail.', chrome_atom_to_build) logging.info('Deleting pin file at %s and proceeding.', CHROMEPIN_MASK_PATH) osutils.SafeUnlink(CHROMEPIN_MASK_PATH) else: raise kwargs = {} if self._chrome_rev == constants.CHROME_REV_SPEC: kwargs['revision'] = self.chrome_version logging.PrintBuildbotStepText('revision %s' % kwargs['revision']) else: if not self.chrome_version: self.chrome_version = self._run.DetermineChromeVersion() kwargs['tag'] = self.chrome_version logging.PrintBuildbotStepText('tag %s' % kwargs['tag']) useflags = self._run.config.useflags commands.SyncChrome(self._build_root, self._run.options.chrome_root, useflags, **kwargs) if (self._chrome_rev and not chrome_atom_to_build and self._run.options.buildbot and self._run.config.build_type == constants.CHROME_PFQ_TYPE): logging.info('Chrome already uprevved. Nothing else to do.') raise failures_lib.ExitEarlyException( 'SyncChromeStage finished and exited early.')
def PerformStage(self): chrome_atom_to_build = None if self._chrome_rev: if (self._chrome_rev == constants.CHROME_REV_SPEC and self._run.options.chrome_version): self.chrome_version = self._run.options.chrome_version logging.info('Using chrome version from options.chrome_version: %s', self.chrome_version) else: self.chrome_version = self._GetChromeVersionFromMetadata() if self.chrome_version: logging.info('Using chrome version from the metadata dictionary: %s', self.chrome_version) # Perform chrome uprev. try: chrome_atom_to_build = commands.MarkChromeAsStable( self._build_root, self._run.manifest_branch, self._chrome_rev, self._boards, chrome_version=self.chrome_version) except commands.ChromeIsPinnedUprevError as e: # If uprev failed due to a chrome pin, record that failure (so that the # build ultimately fails) but try again without the pin, to allow the # slave to test the newer chrome anyway). chrome_atom_to_build = e.new_chrome_atom if chrome_atom_to_build: results_lib.Results.Record(self.name, e) logging.PrintBuildbotStepFailure() logging.error('Chrome is pinned. Unpinning chrome and continuing ' 'build for chrome atom %s. This stage will be marked ' 'as failed to prevent an uprev.', chrome_atom_to_build) logging.info('Deleting pin file at %s and proceeding.', CHROMEPIN_MASK_PATH) osutils.SafeUnlink(CHROMEPIN_MASK_PATH) else: raise kwargs = {} if self._chrome_rev == constants.CHROME_REV_SPEC: kwargs['revision'] = self.chrome_version logging.PrintBuildbotStepText('revision %s' % kwargs['revision']) else: if not self.chrome_version: self.chrome_version = self._run.DetermineChromeVersion() kwargs['tag'] = self.chrome_version logging.PrintBuildbotStepText('tag %s' % kwargs['tag']) useflags = self._run.config.useflags git_cache_dir = ( self._run.options.chrome_preload_dir or self._run.options.git_cache_dir) commands.SyncChrome(self._build_root, self._run.options.chrome_root, useflags, git_cache_dir=git_cache_dir, **kwargs)
def run(self): """Main function for tee subprocess.""" failed = True input_fd = None try: signal.signal(signal.SIGINT, _TeeProcessSignalHandler) signal.signal(signal.SIGTERM, _TeeProcessSignalHandler) # Cleanup every fd except for what we use. self._CloseUnnecessaryFds() # Read from the pipe. input_fd = self._reader_pipe # Create list of files to write to. # Not passing 3 argument (0) for unbuffered output because this is not # supported in Python 3 and there are issues in Python 2 -- see # https://bugs.python.org/issue17404. output_files = [os.fdopen(sys.stdout.fileno(), 'w')] for filename in self._output_filenames: output_files.append(open(filename, 'w')) # Send all data from the one input to all the outputs. _tee(input_fd, output_files, self._complain) failed = False except ToldToDie: failed = False except Exception as e: tb = traceback.format_exc() logging.PrintBuildbotStepFailure(self._error_handle) self._error_handle.write( 'Unhandled exception occured in tee:\n%s\n' % (tb, )) # Try to signal the parent telling them of our # imminent demise. finally: # Close input. if input_fd: os.close(input_fd) if failed: try: os.kill(self.master_pid, signal.SIGTERM) except Exception as e: self._error_handle.write('\nTee failed signaling %s\n' % e) # Finally, kill ourself. # Specifically do it in a fashion that ensures no inherited # cleanup code from our parent process is ran- leave that to # the parent. # pylint: disable=protected-access os._exit(0)
def _HandleExceptionAsError(cls, exc_info): """Handle an exception as an error, but ignore stage retry settings. Meant as a helper for _HandleStageException code only. Args: exc_info: A (type, value, traceback) tuple as returned by sys.exc_info(). Returns: Result tuple of (exception, description, retrying). """ # Tell the user about the exception, and record it. retrying = False description = cls._StringifyException(exc_info) logging.PrintBuildbotStepFailure() logging.error(description) return (exc_info[1], description, retrying)
def kill_us(sig_num, frame): # While this SystemExit *should* crash it's way back up the # stack to our exit handler, we do have live/production code # that uses blanket except statements which could suppress this. # As such, keep scheduling alarms until our exit handler runs. # Note that there is a potential conflict via this code, and # RunCommand's kill_timeout; thus we set the alarming interval # fairly high. signal.alarm(60) # The cbuildbot stage that gets aborted by this timeout should be treated as # failed by buildbot. error_message = ("Timeout occurred- waited %i seconds, failing." % max_run_time) if display_message: error_message += ' Timeout reason: %s' % display_message logging.PrintBuildbotStepFailure() logging.error(error_message) raise SystemExit(error_message)
def PerformStage(self): android_atom_to_build = None if self._android_rev: self.android_version = self._GetAndroidVersionFromMetadata() if self.android_version: logging.info( 'Using Android version from the metadata dictionary: %s', self.android_version) try: android_atom_to_build = commands.MarkAndroidAsStable( self._build_root, self._run.manifest_branch, self._boards, android_version=self.android_version) except commands.AndroidIsPinnedUprevError as e: # If uprev failed due to a pin, record that failure (so that the # build ultimately fails) but try again without the pin, to allow the # slave to test the newer version anyway). android_atom_to_build = e.new_android_atom if android_atom_to_build: results_lib.Results.Record(self.name, e) logging.PrintBuildbotStepFailure() logging.error( 'Android is pinned. Attempting to continue build for ' 'Android atom %s anyway but build will ultimately ' 'fail.', android_atom_to_build) logging.info('Deleting pin file at %s and proceeding.', ANDROIDPIN_MASK_PATH) osutils.SafeUnlink(ANDROIDPIN_MASK_PATH) else: raise if (self._android_rev and not android_atom_to_build and self._run.options.buildbot and self._run.config.build_type == constants.ANDROID_PFQ_TYPE): logging.info('Android already uprevved. Nothing else to do.') raise failures_lib.ExitEarlyException( 'UprevAndroidStage finished and exited early.')
def Wait(self): """Wait for the task to complete. Output from the task is printed as it runs. If an exception occurs, return a string containing the traceback. """ try: # Flush stdout and stderr to be sure no output is interleaved. sys.stdout.flush() sys.stderr.flush() # File position pointers are shared across processes, so we must open # our own file descriptor to ensure output is not lost. self._WaitForStartup() silent_death_time = time.time() + self.SILENT_TIMEOUT results = [] with open(self._output.name, 'r') as output: pos = 0 running, exited_cleanly, task_errors, all_errors = (True, False, [], []) while running: # Check whether the process is still alive. running = self.is_alive() try: errors, results = \ self._queue.get(True, self.PRINT_INTERVAL) if errors: task_errors.extend(errors) all_errors.extend(errors) running = False exited_cleanly = True except Queue.Empty: pass if not running: # Wait for the process to actually exit. If the child doesn't exit # in a timely fashion, kill it. self.join(self.EXIT_TIMEOUT) if self.exitcode is None: msg = '%r hung for %r seconds' % (self, self.EXIT_TIMEOUT) all_errors.extend( failures_lib.CreateExceptInfo(ProcessExitTimeout(msg), '')) self._KillChildren([self]) elif not exited_cleanly: msg = ('%r exited unexpectedly with code %s' % (self, self.exitcode)) all_errors.extend( failures_lib.CreateExceptInfo(ProcessUnexpectedExit(msg), '')) # Read output from process. output.seek(pos) buf = output.read(_BUFSIZE) if len(buf) > 0: silent_death_time = time.time() + self.SILENT_TIMEOUT elif running and time.time() > silent_death_time: msg = ('No output from %r for %r seconds' % (self, self.SILENT_TIMEOUT)) all_errors.extend( failures_lib.CreateExceptInfo(ProcessSilentTimeout(msg), '')) self._KillChildren([self]) # Read remaining output from the process. output.seek(pos) buf = output.read(_BUFSIZE) running = False # Print output so far. while len(buf) > 0: sys.stdout.write(buf) pos += len(buf) if len(buf) < _BUFSIZE: break buf = output.read(_BUFSIZE) # Print error messages if anything exceptional occurred. if len(all_errors) > len(task_errors): logging.PrintBuildbotStepFailure() msg = '\n'.join(x.str for x in all_errors if x) logging.warning(msg) traceback.print_stack() sys.stdout.flush() sys.stderr.flush() # Propagate any results. for result in results: results_lib.Results.Record(*result) finally: self.Cleanup(silent=True) # If an error occurred, return it. return all_errors