def testChildExits(self): """Create a child and a grandchild. The child should die with the parent.""" def GrandChild(): parallel.ExitWithParent() time.sleep(9) def Child(queue): grand_child = multiprocessing.Process(target=GrandChild) grand_child.start() queue.put(grand_child.pid) time.sleep(9) with parallel.Manager() as manager: q = manager.Queue() child = multiprocessing.Process(target=lambda: Child(q)) child.start() grand_child_pid = q.get(timeout=1) # Before we kill the child, the grandchild should be running: self.assertTrue(os.path.isdir('/proc/%d' % grand_child_pid)) os.kill(child.pid, signal.SIGKILL) # (shortly) after we kill the child, the grandchild should kill itself. # We can't use os.waitpid because the grandchild process is not a child # process of ours. Just wait 20 seconds - this should be enough even if the # machine is under load. timeout_util.WaitForReturnTrue( lambda: not os.path.isdir('/proc/%d' % grand_child_pid), 20, period=0.05)
def _WaitUntilStarted(self): """Wait until the devserver has started.""" if not self.port: self._ReadPortNumber() try: timeout_util.WaitForReturnTrue(self.IsReady, timeout=self.DEV_SERVER_TIMEOUT, period=5) except timeout_util.TimeoutError: self.terminate() raise DevServerStartupError('Devserver did not start')
def _WaitForSigningResults(self, instruction_urls_per_channel, channel_notifier=None): """Do the work of waiting for signer results and logging them. Args: instruction_urls_per_channel: push_image data (see _WaitForPushImage). channel_notifier: Method to call with channel name when ready or None. Raises: ValueError: If the signer result isn't valid json. RunCommandError: If we are unable to download signer results. """ gs_ctx = gs.GSContext(dry_run=self._run.options.debug) try: logging.info('Waiting for signer results.') timeout_util.WaitForReturnTrue( self._CheckForResults, func_args=(gs_ctx, instruction_urls_per_channel, channel_notifier), timeout=self.SIGNING_TIMEOUT, period=self.SIGNING_PERIOD) except timeout_util.TimeoutError: msg = 'Image signing timed out.' logging.error(msg) logging.PrintBuildbotStepText(msg) raise SignerResultsTimeout(msg) # Log all signer results, then handle any signing failures. failures = [] for url_results in self.signing_results.values(): for url, signer_result in url_results.items(): result_description = os.path.basename(url) logging.PrintBuildbotStepText(result_description) logging.info('Received results for: %s', result_description) logging.info(pformat.json(signer_result)) status = self._SigningStatusFromJson(signer_result) if status != constants.SIGNER_STATUS_PASSED: failures.append(result_description) logging.error('Signing failed for: %s', result_description) details = signer_result.get('status', {}).get('details') if details: logging.info('Details:\n%s', details) if failures: logging.error('Failure summary:') for failure in failures: logging.error(' %s', failure) raise SignerFailure(', '.join([str(f) for f in failures]))
def _ReadPortNumber(self): """Read port number from file.""" if not self.is_alive(): raise DevServerStartupError('Devserver terminated unexpectedly!') try: timeout_util.WaitForReturnTrue(os.path.exists, func_args=[self.port_file], timeout=self.DEV_SERVER_TIMEOUT, period=5) except timeout_util.TimeoutError: self.terminate() raise DevServerStartupError('Devserver portfile does not exist!') self.port = int(osutils.ReadFile(self.port_file).strip())
def RemoteReboot(self, timeout_sec=REBOOT_MAX_WAIT): """Reboot the remote device.""" logging.info('Rebooting %s...', self.remote_host) if self.username != ROOT_ACCOUNT: self.RemoteSh('sudo sh -c "touch %s && sudo reboot"' % REBOOT_MARKER) else: self.RemoteSh('touch %s && reboot' % REBOOT_MARKER) time.sleep(CHECK_INTERVAL) try: timeout_util.WaitForReturnTrue(self._CheckIfRebooted, timeout_sec, period=CHECK_INTERVAL) except timeout_util.TimeoutError: cros_build_lib.Die('Reboot has not completed after %s seconds; giving up.' % (timeout_sec,))
def test010SubmitBatchUsingGit(self): project = self.createProject('test012') helper = self._GetHelper() repo = self.cloneProject(project, 'p1') initial_patch = self.createPatch(repo, project, msg='Init') helper.SetReview(initial_patch.gerrit_number, labels={'Code-Review': '+2'}) helper.SubmitChange(initial_patch) # GoB does not guarantee that the change will be in "merged" state # atomically after the /Submit endpoint is called. timeout_util.WaitForReturnTrue( lambda: helper.IsChangeCommitted(initial_patch.gerrit_number), timeout=60) patchA = self.createPatch(repo, project, msg='Change A', filename='a.txt') osutils.WriteFile(os.path.join(repo, 'aoeu.txt'), 'asdf') git.RunGit(repo, ['add', 'aoeu.txt']) git.RunGit(repo, ['commit', '--amend', '--reuse-message=HEAD']) sha1 = git.RunGit(repo, ['rev-list', '-n1', 'HEAD']).output.strip() patchA.sha1 = sha1 patchA.revision = sha1 patchB = self.createPatch(repo, project, msg='Change B', filename='b.txt') pool = validation_pool.ValidationPool(overlays=constants.PUBLIC, build_root='', build_number=0, builder_name='', is_master=False, dryrun=False) reason = "Testing submitting changes in batch via Git." by_repo = {repo: {patchA: reason, patchB: reason}} pool.SubmitLocalChanges(by_repo) self.assertTrue(helper.IsChangeCommitted(patchB.gerrit_number)) self.assertTrue(helper.IsChangeCommitted(patchA.gerrit_number)) for patch in [patchA, patchB]: self.assertTrue(helper.IsChangeCommitted(patch.gerrit_number))
def _WaitUntilStarted(self): """Wait until the nebraska has started.""" if not self._port: self._ReadPortNumber() try: timeout_util.WaitForReturnTrue(self.IsReady, timeout=self.NEBRASKA_TIMEOUT, period=5) except timeout_util.TimeoutError: raise NebraskaStartupError('Nebraska did not start.') self._pid = int( self._RemoteCommand(['cat', self._pid_file], capture_output=True).output.strip()) logging.info('Started nebraska with pid %s', self._pid)
def AwaitReboot(self, old_boot_id, timeout_sec=REBOOT_MAX_WAIT): """Await reboot away from old_boot_id. Args: old_boot_id: The boot_id that must be transitioned away from for success. timeout_sec: How long to wait for reboot. Returns: True if the device has successfully rebooted. """ try: timeout_util.WaitForReturnTrue(lambda: self.CheckIfRebooted(old_boot_id), timeout_sec, period=CHECK_INTERVAL) except timeout_util.TimeoutError: return False return True
def _ReadPortNumber(self): """Read port number from file.""" if not self.is_alive(): raise DevServerStartupError('Devserver is dead and has no port number') try: timeout_util.WaitForReturnTrue(os.path.exists, func_args=[self.port_file], timeout=self.DEV_SERVER_TIMEOUT, period=5) except timeout_util.TimeoutError: self.terminate() raise DevServerStartupError('Timeout (%s) waiting for devserver ' 'port_file' % self.DEV_SERVER_TIMEOUT) self.port = int(osutils.ReadFile(self.port_file).strip())
def _VerifyBootExpectations(self, expected_kernel_state, rollback_message): """Verify that we fully booted given expected kernel state. It verifies that we booted using the correct kernel state, and that the OS has marked the kernel as good. Args: expected_kernel_state: kernel state that we're verifying with i.e. I expect to be booted onto partition 4 etc. See output of _GetKernelState. rollback_message: string to raise as a RootfsUpdateError if we booted with the wrong partition. """ logging.debug('Start verifying boot expectations...') # Figure out the newly active kernel active_kernel_state = self._GetKernelState()[0] # Rollback if (expected_kernel_state and active_kernel_state != expected_kernel_state): logging.debug('Dumping partition table.') self.device.RunCommand(['cgpt', 'show', '$(rootdev -s -d)'], **self._cmd_kwargs) logging.debug('Dumping crossystem for firmware debugging.') self.device.RunCommand(['crossystem', '--all'], **self._cmd_kwargs) raise RootfsUpdateError(rollback_message) # Make sure chromeos-setgoodkernel runs try: timeout_util.WaitForReturnTrue( lambda: (self._GetKernelTries(active_kernel_state) == 0 and self._GetKernelSuccess(active_kernel_state)), self.KERNEL_UPDATE_TIMEOUT, period=5) except timeout_util.TimeoutError: services_status = self.device.RunCommand( ['status', 'system-services'], capture_output=True, log_output=True).output logging.debug('System services_status: %r' % services_status) if services_status != 'system-services start/running\n': event = ('Chrome failed to reach login screen') else: event = ('update-engine failed to call ' 'chromeos-setgoodkernel') raise RootfsUpdateError('After update and reboot, %s ' 'within %d seconds' % (event, self.KERNEL_UPDATE_TIMEOUT))
def _ReadPortNumber(self): """Reads the port number from the port file on the remote device.""" if not self.is_alive(): raise NebraskaStartupError( 'Nebraska is not alive, so no port file yet!') try: timeout_util.WaitForReturnTrue(self._PortFileExists, period=5, timeout=self.NEBRASKA_TIMEOUT) except timeout_util.TimeoutError: self.terminate() raise NebraskaStartupError( 'Timeout (%s) waiting for remote nebraska' ' port_file' % self.NEBRASKA_TIMEOUT) self._port = int( self._RemoteCommand(['cat', self._port_file], capture_output=True).output.strip())
def _ReadPortNumber(self): """Read port number from file.""" if not self.is_alive(): raise DevServerStartupError('Devserver terminated unexpectedly!') def PortFileExists(): result = self._RunCommand(['test', '-f', self.port_file], error_code_ok=True) return result.returncode == 0 try: timeout_util.WaitForReturnTrue(PortFileExists, timeout=self.DEV_SERVER_TIMEOUT, period=5) except timeout_util.TimeoutError: self.terminate() raise DevServerStartupError('Devserver portfile does not exist!') self.port = int( self._RunCommand(['cat', self.port_file], capture_output=True).output.strip())
def WaitForAFDOPerfData(cpv, arch, buildroot, gs_context, timeout=constants.AFDO_GENERATE_TIMEOUT): """Wait for AFDO perf data to show up (with an appropriate timeout). Wait for AFDO 'perf' data to show up in GS and copy it into a temp directory in the buildroot. Args: arch: architecture we're going to build Chrome for. cpv: CPV object for Chrome. buildroot: buildroot where AFDO data should be stored. gs_context: GS context to retrieve data. timeout: How long to wait total, in seconds. Returns: True if found the AFDO perf data before the timeout expired. False otherwise. """ try: timeout_util.WaitForReturnTrue( CheckAFDOPerfData, func_args=(cpv, arch, gs_context), timeout=timeout, period=constants.SLEEP_TIMEOUT) except timeout_util.TimeoutError: logging.info('Could not find AFDO perf data before timeout') return False url = GetAFDOPerfDataURL(cpv, arch) dest_dir = AFDO_BUILDROOT_LOCAL % {'build_root': buildroot} dest_path = os.path.join(dest_dir, url.rsplit('/', 1)[1]) gs_context.Copy(url, dest_path) UncompressAFDOFile(dest_path, buildroot) logging.info('Retrieved AFDO perf data to %s', dest_path) return True
def _WaitForOperation(self, operation, get_operation_request, timeout_sec=None, timeout_handler=None): """Waits until timeout or the request gets a response with a 'DONE' status. Args: operation: The GCE operation to wait for. get_operation_request: The HTTP request to get the operation's status. This request will be executed periodically until it returns a status 'DONE'. timeout_sec: The maximum number of seconds to wait for. timeout_handler: A callable to be executed when times out. Raises: Error when timeout happens or the operation fails. """ def _IsDone(): result = get_operation_request.execute() if result['status'] == 'DONE': if 'error' in result: raise Error(result['error']) return True return False try: timeout = timeout_sec or self.DEFAULT_TIMEOUT_SEC logging.info( 'Waiting up to %d seconds for operation [%s] to complete...', timeout, operation) timeout_util.WaitForReturnTrue(_IsDone, timeout, period=1) except timeout_util.TimeoutError: if timeout_handler: timeout_handler() raise Error('Timeout wating for operation [%s] to complete' % operation)
def _ReadPortNumber(self): """Read port number from file.""" if not self.is_alive(): raise DevServerStartupError('Devserver not alive ' 'therefore no port number') def PortFileExists(): result = self._RunCommand(['test', '-f', self.port_file], error_code_ok=True) return result.returncode == 0 try: timeout_util.WaitForReturnTrue(PortFileExists, timeout=self.DEV_SERVER_TIMEOUT, period=5) except timeout_util.TimeoutError: self.terminate() raise DevServerStartupError( 'Timeout (%s) waiting for remote devserver' ' port_file' % self.DEV_SERVER_TIMEOUT) self.port = int( self._RunCommand(['cat', self.port_file], capture_output=True).output.strip())